{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Handwritten Digit Recognition\n",
    "\n",
    "This tutorial guides you through a classic computer vision application: identify hand written digits with neural networks. \n",
    "\n",
    "## Load data\n",
    "\n",
    "We first fetch the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset, which is a commonly used dataset for handwritten digit recognition. Each image in this dataset has been resized into 28x28 with grayscale value between 0 and 254. The following codes download and load the images and the according labels into `numpy`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import urllib\n",
    "import gzip\n",
    "import struct\n",
    "def download_data(url, force_download=True): \n",
    "    fname = url.split(\"/\")[-1]\n",
    "    if force_download or not os.path.exists(fname):\n",
    "        urllib.urlretrieve(url, fname)\n",
    "    return fname\n",
    "\n",
    "def read_data(label_url, image_url):\n",
    "    with gzip.open(download_data(label_url)) as flbl:\n",
    "        magic, num = struct.unpack(\">II\", flbl.read(8))\n",
    "        label = np.fromstring(flbl.read(), dtype=np.int8)\n",
    "    with gzip.open(download_data(image_url), 'rb') as fimg:\n",
    "        magic, num, rows, cols = struct.unpack(\">IIII\", fimg.read(16))\n",
    "        image = np.fromstring(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)\n",
    "    return (label, image)\n",
    "\n",
    "path='http://yann.lecun.com/exdb/mnist/'\n",
    "(train_lbl, train_img) = read_data(\n",
    "    path+'train-labels-idx1-ubyte.gz', path+'train-images-idx3-ubyte.gz')\n",
    "(val_lbl, val_img) = read_data(\n",
    "    path+'t10k-labels-idx1-ubyte.gz', path+'t10k-images-idx3-ubyte.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We plot the first 10 images and print their labels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAABVCAYAAACow4a8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3VdsXFd++PHv9OEMp5DDYe+9iTRJSVShKIuSaVuWLFnS\n2lo5yXrhwEiCIAiCPAUB8pCXJA9BFkGATbDe2Ju1196VVy7yqkukKatQlZ1ir8M65LANy7T/g3Dv\n37QsV5oz9J4PICxWkoe/Gd25v3vO+Z3fAUEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQ\nBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQ\nBEEQBEEIPYog//xAkH/+1/Fln1Goxy9iD57Hxb+RY4fQj1/EHjwbOf4fauxfSPl9RCEIgiAIQmgS\niV8QBEEQ/oiIxC8IgiAIf0TUwQ5AWC08PBy73U5ycjIREREATE9P09/fz+TkJPPz80GOUBD+OKlU\nKvLz80lMTMRsNuPxeHjw4AE9PT0sLS0RCIT6UrCw0eTl5ZGeng7AvXv3cDgca/K6P5jEr1Csrm9Q\nqVSoVCoAlEolfr8fj8eDWq1GrVajVD6c7PD7/fh8PjweD36/f93j/iylUklSUhKVlZUcOXKE4uJi\nAO7fv8/Jkyepq6vjwYMHQY3x21IqlahUKjQaDV6vF6/XG/TP+6tI14pa/fBrIl0nPp9P3OTXmPT9\nValU8nczlK4RhUKBXq/npZde4tChQ+Tm5uJyufjZz37GG2+8gcPhCOlrQvpMFQoFSqUStVot3x8f\nx+/34/V6WVlZWY8QvxbpPqJWq/F6vXg8nmCH9L1QKBRotVqOHj3Kn/zJn+D3+/n7v/97kfglCoUC\nlUpFWFiYfHEDbNmyhU2bNqHVarHZbIyOjvLOO+9w4MABduzYQXJyMn6/n4GBAerr6zl9+jRDQ0NB\nex86nY6UlBROnDjBoUOHiIuLIzw8HIDS0lJiY2MJCwvbsIk/ISGBzZs3c+zYMWpqavj4448ZHR0N\nmRv7F6msrKSqqory8nJUKhUtLS288847tLW1MTU1FezwfjC0Wi16vR6dTkdhYSHJyclotVrOnz/P\n0NAQPp8v2CESFhZGYmIiRUVFJCYmolQqMZlMJCYmkpiYGNLXskqlIiYmBoPBgMFgICsri507d/LE\nE0986X/ncDj45JNP+OUvfxkSD7sqlYr4+HiKi4vZv38/H330EWfOnAlqTN+XyMhIjh8/TlVVFSaT\nic7OTpaWltbs9TdM4tdoNBgMBlQqFWazGbvdjt1ux2g0YjAYiImJQavVyn8/JyeH9PR01Go1RqOR\nkZERPB4PTz/9NGVlZdjtdhYWFlCr1TQ0NKx6aFhvZrOZtLQ0Dh48SHV1NXl5efIsRSAQwGKxYLFY\n5GnGiYkJVlZW1vyLaDKZ5M9xamqK0dHRNXttu91OcXExRUVFdHR0EBYWtmavvdbCw8PJyMigurqa\nZ599loKCApRKJVqtlgsXLqy6zoRvR6/XExkZSWFhIfHx8dhsNkwmE2lpaURHR6NSqdDpdNTU1NDc\n3BzscAkEAvL3ER4OOKR7ksFgCHJ0j1IoFFitVuLj48nIyCAvLw+LxYJeryc1NZVNmzaRkZHxpa8x\nOTmJxWJhenqa+vp6hoeH8Xq96/QOVlMoFOh0OkpKSnj55ZcpKyujra2NsLCwH9wyS2pqKhUVFfLM\n0srKCu3t7czMzKzZz9gQiV+r1RITE0NmZiYajYaEhATy8/PJzc0lKioKs9lMamrqY5PJ5OQkgUCA\n559/nqSkJHmk73Q6aWpqorGxcd3XzhUKhXwxp6ens2/fPn7605+SmJiISqUiEAjg9XpZWFhgYWEB\nu91OamoqO3fu5M6dO4yMjLCwsLCmMdlsNrZu3UpERASNjY1rmvgjIiJITEzE7XazvLy8Zq/7fbBa\nrezevZvKykry8/MfWUYKNo1Gg06nw2g0otVq0Wg0KJVKFAoFXq+X+fl5vF6vfH253W6WlpaC/rlL\nD08mk4nY2Fjy8/M5duwY+fn5xMbGYjKZgIffDb/fL6+jd3R04PF4gnpz93g8uFwuHA4HLpcLi8US\ntFi+DqVSSVZWFpWVlezdu5fCwkIsFgtKpRKdTidP83/ZZ2qz2SgvL8disfBv//ZvuFyuNU0+35RO\np6O0tJRjx46xvLxMREQEFotlTUfCoaCwsJAf//jHbNmyBb1eT0tLCzdv3mRycnLNfsaGSPxJSUk8\n99xz/OVf/iUajQatVotOp0Ov18tr+RqN5gv/20AgQH19Pbdu3WJhYQGNRsPKygrT09Ny0VxHRweL\ni4vr+p6USiVhYWHk5OTwwgsv8NJLLxEXFyevJwMsLCzQ1NTE1atXOXHiBDt27CAhIYEzZ87w3nvv\nUV9fv6YxRUREUFpaKifourq6NXldhUKByWQiISGByMhIdDpdyCXTz9JqtURHR2M0GoM6E/Q40dHR\nFBYWsmvXLnJzc0lISMBsNqNUKhkdHeXChQuMjY2h1+vJzc3l+vXr3LlzJ+jLROHh4fLM1vbt28nN\nzSUiIgK9Xo9SqWRpaUmuwzEajWRnZ1NQUEBiYiJDQ0NBXWv2+Xy43W6cTueGKLBVq9UcPHiQ/fv3\nk5GRId8rgW90TVutVoqKiigoKKC7uzuoiV8iPdBarVasVisTExMhsRy0ViIjI0lPT0en0zE3N0dn\nZydnz57F6XSu2c/YEIl/cXERhUJBQkICYWFhjy1KWVxcxO124/F4CA8Px2Aw4Pf7uXv3Lh9++KE8\nTeX1ellcXGR5eZn5+fmgfJEzMzPZv38/hYWFlJSUkJSUhFqtXpUQpRvizMwMTqcTu91OSkoKRUVF\na5aUP8tut7Nt2zZ8Ph96vX5NXlOhUBAeHk5ycjKZmZksLi4yPj4uz8KEmvj4eEpKSti+fTuxsbH4\nfD4WFxe5cOECFy5coKOjI2g3fo1GQ25uLrt27aKqqorMzEz5AWV5eRmVSoXVaiU8PJz5+XnUajXR\n0dHYbDaUSmVQEr80wtyxYwebN28mLy9PXsePiIhAqVSysrJCT08Pp06dwuPxsGnTJg4fPkx4eDg2\nm42oqChGRkbWPfbPv4+wsDAiIyMxGo1BjeXrCAQCzM3N4fF45JkUv9+P2+2mt7d31Sg5EAig1+uJ\niYmRlwMkKpVKXs543OAqGBQKxarC21CmVCpJSUmhoKCAmJgYzpw5w8TExCOFidKukcLCQmw2G16v\nl5s3b3Lu3DmcTueaPviG/qcGuFwuBgcH6ejoICYmBrVajcfjwW63o9PpCAQCeDweOjs7aWtrY3Fx\nkdjYWBISEoiKiqKjo4N79+4F+23IDAYDhYWF/OQnPyE9PR2j0UggEJAfRAKBAFFRUeh0OtRqNW63\nm87OTiwWC4mJiXJtw1ozm83k5OTQ19e3Zq+pUqkoKCigqKiI2NhYbt++zcjICC6Xa81+xlpQq9WE\nh4dTVlZGdXU1JSUlmEwm5ubm6Onp4eTJk1y8eFG+ma4naYRjt9vZt28fBw4cYPv27fj9flwuFx0d\nHfT29qJQKLDb7WzatInw8HD5QdLtdtPU1LSuMUv0ej1JSUkcOHCAZ555hvT0dFQqFQqFQl7O6unp\n4cKFC7z++uuo1WoOHz7M4cOHUalU6PV6DAZD0GdepOsjOjoas9ks/75er8dkMhEWFobb7Q6Zkaff\n76ehoQG73S4/YHu9XmZmZrh79y5zc3Or/r7RaCQ9PZ3y8nLS09NX7ayYn5/H5XLhdrvX/X08jrQ7\nIdjXxdehVqspLy/nwIEDJCUl0dHRwcLCwiOzJxqNhp07d1JaWopOp6Onp4crV65QW1u75rUVGyLx\nLy4u0tzczJtvvsm2bdtQKBRMTExw5MgREhIS8Pl8uFwu3nvvPX7+85+ztLS0qoo81Cqwk5KSKCgo\nICsrS5729ng8jI2N0dDQgMfj4dChQ/j9fpxOJy0tLczNzaHT6UhKSvrepsmluoO1fH2dTsef/dmf\nsW/fPjweDw6H45GbTigICwujuLiYY8eOcfDgQUwmE0qlkrGxMd5//32amppwOp1BmaVQKpVER0ez\ndetWjh8/zqZNm+TYLly4wOnTp7l9+zYej4fi4mL+6Z/+iby8PLkPxPDwMMPDw+seNzyctqyqqmLH\njh1kZGSsmq3z+/0sLCxw8uRJ3njjDQYHB4mLiwtKnF9F2jlkNptX1RJFR0eTmZlJW1sbDocjZJYB\nvF4vtbW1NDY28qtf/Qp4uFyxvLy8arQpXc8ajYbw8HD++Z//mcTERHnUPzc3R1dXF83NzQwODgbn\nzXyBQCAg17qEOq1Wy5EjR9i7dy9zc3NkZ2czODj4SOJXq9Xs2bOH0tJSFhcXef/997ly5Qr9/f1r\nvmNkQyT+QCCAw+HgzJkz8sjF6/USHx/Pjh07MBqN1NbWcv/+faampuTivYWFBfr7++nu7g7yO3jI\nYDAQHx/Pn/7pn7J//360Wi0KhYLZ2Vk6Ojr43//9X5aXl0lKSkKv13Pv3j1u3LhBa2srfX195OXl\ncejQIZKSksjOziY5OZmhoaE1uShiY2OJj49f8/V3aX3fYDCwtLREc3MzY2Nja/b6a8VsNrN7927y\n8/PlpD87O0t7ezsffvghg4ODQUn6KpWKzZs3U1VVxbPPPktmZibLy8v09vbyq1/9ivr6enp6enA6\nnWRnZ1NUVERSUhJGoxG/38/Kygq3b9/mzp076x47PFwj3rVrF3FxcXLR6vLyMjMzM/T39/Phhx9y\n8eJFRkdH8fl88m6dULOyssLExAT3798nISGBvLw8AHJzc1laWmJ2dpaLFy+GTOIHWF5exul0yg/a\ngUAAn8/HysrKI/eMsLAw8vPzsdvtqx7OlpeXGR8fD8pM11eJiooiMTGR+/fvh8xMyxeRtpxLv/R6\n/SPLJmazmYyMDHnbpfRvJP1aaxsi8QPMz8/T1dUlJzqDwUB3dze5ubkYDIZVTWECgQButxu32x0y\nT6nSPuDdu3fLVbbSTXBwcJDr169z5swZNBoNqampdHV1ce/ePfr6+uQpNqfTiVKpJCIigqKiIkpL\nSxkZGVmTCyM1NZX09HS5QnwtSKOk8PBwtFotbreb7u7ukJuBUavVREZGsm3bNpKTk+WtlJ2dndTX\n19PW1haUm57NZiMnJ4fnnnuOvXv3UlpaitPppLm5mdraWvmBRIotPz+f3bt3Y7fb5S2ZLS0t3Llz\nh97e3nWPHx4mjuHhYXp6epiamsLpdOJ0OhkbG6Orq4s//OEPDA8Py8W1ERERREZGBiXWLyNNk1++\nfJnk5GQ58dvtdrKyskhLS5P7boSKQCDAysrKV64NR0VFkZ+fz969e0lOTl6V+CcmJqitrWViYiLo\nfQqkLZV+vx+lUonNZiMuLi6kp/ttNhubNm2Sr+nFxUVcLteqGgupP8GePXuIjo7G7/czMzPzhbMC\na2XDJH6J9IHp9XpmZmZYWFhAr9ezY8cO6uvruXjxIl6vN+QKx6KioigrK+P48eNkZGTg9/vlwsLG\nxkbq6uqYmppidnaWzs7Ox76OtLZVUlKCw+Hg3Llza5KUsrKyyM7OXtU/4LsKCwsjNjZWrtz2eDyM\nj4+H1KhImpGQll+ioqLkgr5r165RU1MTlBueRqMhPz+f1157jT179hAXF4fb7aahoYF3332XkydP\nymvK0oiiqKiIyspKdDodPp+P/v5+3nzzTZqbm4O2PjswMMB//Md/cODAAcxmM3fv3qWvr08eRX6e\nyWQKuQQq8Xg8nD17loKCAl588cVgh/OdSKNPtVrNE088QXV1NdXV1aSkpMiJNBAI0N3dzS9+8YuQ\nWN+XZitWVlbQ6XSYzWZsNlvI7hBSKBRkZ2fz6quvkpOTA8DIyAgPHjxYVaFvMBgoKCiQd3bNzc3R\n0dHBJ598wsDAwPcS24ZL/JLFxUV+//vfyyPpyMhICgoKKCkpoampad235z2OVH26Z88efvzjH7Np\n0yaMRiOtra2cOnWKzs5ORkZGmJiY+Eb7rI1GIyaTac0u+ujoaLkQyOFwrMme0by8PF577TXS09Px\n+/3Mzc2xsLAQtCYgn6dWq7HZbBw6dIiXXnoJm80GPCwmvXnzJp988gmtra1BSfzV1dU8//zz7N27\nF4vFgsvloqurizfeeINr166xuLiI3++XK/kLCwvJysoiLCwMhUJBa2sr586d4+LFi2u6//eb8ng8\nTE5Ocvr0adRqNXNzcywuLj52FJqcnExycvI6R/nNffZ7F6qJ58tkZ2ezY8cOKisrycjIIC4ujqio\nqFUV/fPz88zOzoZE62RpFndycpLR0VHi4+ODGs/XYbFYyMrKoqKiArPZzK1bt3j77bcZGBhYda8/\nePAgx44dIzs7G7Vazfvvv8/Pfvaz77UN9IZN/FI1cE1NDTExMTz//POUlJRw/PhxcnJyGB0dlZ+u\ngrk2pdFoiIuL44knnqC0tJTIyEju3LnD2bNn+fDDD3E4HPJN8JskxC/rXfBtSC1TfT4fg4ODTExM\nfOPXUKvVchvTlJQUKioq2LNnDzabDYfDwa1bt9Z8W8p3odVqycnJYdu2bWzevBm9Xo/b7aanp4eP\nPvqIxsbGdd+3LDW52b59OxUVFcTExMid0z744AOuXbsmF+qZzWaKi4vZunUr+fn5bNq0Cb/fz/j4\nOLW1tVy4cAGHwxHU9U+/3y9P939WfHw8aWlpcoW/VOVfWFhIdHQ0gUCAhYUFxsbGGB4eDrn15VCb\nUfw8hUKB0WgkNjaW1NRUoqOjVxXC5eTkUFZWxqZNm7BYLGg0GrlxktPppKOjg87OTmpqar6XLqHf\nhlSc6Ha7QyKex9FqtURERFBVVcX+/fuJiYlhdHSU+vp6amtrcblc+Hw+LBYLBQUFVFdXs2XLFgwG\nA01NTdy4cYP79+9/rzFu2MQvbeG7e/cuHo+HnJwc8vPzSU5OprKykp6eHm7dusXc3ByTk5MsLS0F\n5Qao1+spKSkhNzdX3pt5/vx5fve7333nVqTfx0gjEAgwMTHxSMKTEpI0gyF1/5K21Oh0OgwGA3Fx\ncWzZsoXy8nJ5v7ZSqaS7u5tLly7hdDpDYsQvNemRGskYjUZ8Ph8DAwPcuHGDP/zhD4yPj697XFLz\nmtzcXDIyMvD5fHR2dvLxxx/zy1/+EqPRSExMDEajkcTERF588UWOHDlCREQEKpWK6elpWlpauHz5\nMtevXw+ZoidpOUKaot2+fTtVVVVyrwp4eO0VFRVhtVrlB/vm5ma6u7uDPuL8IlLyCbUkJH3O2dnZ\nlJeXs3v3bvLy8lZtQ7RarVgslkdi93g89PX18e6773Lz5k26urpC7qFLIh3YE0rCwsKIi4ujuLiY\nV155hZ07d6JSqRgeHpZPWJVm6uLj4/nRj37Etm3biI6OZm5ujitXrqzL1tsNm/glc3NzPHjwgLfe\neoujR49SWVkp3zSzs7MxGo1cvnyZ5uZmpqen1/1LarVaOXHiBMXFxSwvLzM2NkZjYyNdXV3f6vWk\nZP99bukzGAyPbJMxGAxkZGSgVquJjY2loKAAm82GwWAgPDycgoICDAaD/FAgtRP2+/3y9svOzs6g\nt42VpKamsmfPHo4ePUpOTo48Jf3ee+/xzjvvfGGDjfUg9YDXarWo1WqWl5f57W9/S21tLXFxcezd\nu5ctW7aQl5dHVFQU0dHRctKHh8sUly9fpqenJ2RmVuDhzFdUVBSZmZkcPXqU7du3k5aWJrcahv/f\nSEbaUfHGG29w4cKFkEz6ocxsNpOZmcnf/M3fsH37drlb5meT5OMSptPppLGxkZMnTzIzMxMy39cv\nIhUOh8JSi1R7JR0gdPz4cflgtUAgQFJSElVVVfj9fk6dOsXKygopKSns3LmT6OhopqenaWpq4vTp\n09/7aB9+AIlf2sNfU1PD8vIyIyMj7Nq1i9jYWNLT0zl06BA2mw273U5tbS2zs7PrdkOMiYmhtLRU\nbk06OzvL/fv3cTgc37q/tPTgMjs7y9TU1Jo9yHi9Xnw+H2q1moqKCiIiIqiurpb/3GAwyOcIhIeH\nExMTg16vl487ViqVjIyMMDIyQn9/P6Ojo+Tn5/PTn/4UpVKJy+Wiv78/6KMHpVKJ2Wxm27ZtnDhx\ngqysLIxGI0tLSzidTnp7e+nt7Q3q9KbP51v17/HUU0+Rk5ODUqkkMzNTbuIkHa0qVTn7/X4mJyf5\n5JNPGBoaCvpIVOp2FxcXR1VVldzESerNr9VqmZ6eJjw8fFW9ijRLERUVhdVqRaVShczMxUYg7aaJ\ni4sjLi7uSw/E+nzSlM5RSElJoaOjIySK+h5HquoPhcQfFhZGQUEBBw8e5Nlnn5WXsXw+H0tLS1it\nVsrKyrDZbKSlpREIBEhLSyMtLQ2dTse9e/f4r//6L1pbW9flM9/wiR8e7rHt7OzE5XLJ+/eLiopI\nS0ujsLCQ8PBwLBYLPp+PW7durVszE7vdTm5uLpGRkWg0Gqampqirq/tGU8jSFGl8fDx2u11eM+3o\n6KCpqWnNboh9fX20trZis9lIT08nJSVl1cOJRqPBbDYzNzfH/Pw8brdb3oUgdVYcGBigp6eHtrY2\neRlAoVDg8/mYn58PiW18arWanJwcKioq2LVrl/z7i4uLtLa2MjQ0FNTCUOlG0dPTQ39/P2lpaTz9\n9NPy0tb8/DzT09PyTdlutxMfHy93GRwaGqK1tZXp6emgvQd4eN1Kp05WVFRw4sQJcnJymJ+fZ2Zm\nhpaWFqanp5mcnKSoqIj8/PxVs0w6nY7y8nIcDgfDw8NMTk4G/aCex9FoNFit1pBpaevxeJiZmaG9\nvR2r1Yrdbmdubu6xD7MWi4XIyEjMZjNms5nExETy8vIYHR0Nie/sZ0k9E3w+n1zVH+ztfOHh4aSm\npsrdKYuKiggEAgwODjI5OYnb7SYzM5O4uDgSEhIoKChArVbLdVUejwe3283Y2BgKhQKtVvu9D05/\nEIlf4nQ6uXHjBg8ePGDr1q0888wzvPzyy6SmphIREUFBQQH/8A//sG6JX6VSodVq5Sc/qdPaN+k7\nrlQqMRqN/OhHP2LXrl14vV5GRkb4+OOP+d3vfrdmI+hTp04xODiIy+UiPz8fs9m86iahUCiYnJyk\nvr6e5uZm+vr6aGhokJdP/H4/Pp9P/t/9+/ezbds2tFotCwsLITPtHBYWxsGDB9m6deuq9+d0Ovnt\nb38b9CNgpRap7777Ln6/n7/+679Gq9Xi8XiYnp6moaGBmpoarl69isvl4sUXX+SVV17BaDQyODhI\ne3t70GsopDqQrKwsDh48yF/91V9hMpno6+vj4sWL3L9/n3v37tHV1YXRaOTv/u7vSE1NlWtIVCoV\nRqORyspKVlZWcDqdXLlyhampqZAY+X++u6XJZKK4uDhk+g/Mzs7S2trKv//7v1NeXk55eTm3b9+W\nj/P+vJ07d7Jv3z62bt0qt0APVWNjY7S2tpKTkxMyx2NLB0/95Cc/kc/38Hg8vP/++9TV1bG8vMxf\n/MVfsHnzZqKiouTrRLqGNBoNe/bsITs7m3/8x3+kpqbmWxVXfxM/qMQvjYadTif19fUYjUaOHTsm\nb31LT0+nuLiYrq6udevmJ/3j+v1+FhcXGRsb+9rT/CqViqysLJ566ikOHDhAeno6k5OTnDt3jsbG\nRubm5tbsSyqd+fz6669jtVof+6WamJjA5XLJI/jHJfSMjAy5OK2joyNoLWM/y2KxkJOTQ2Fh4arW\nsNPT03R3d9PZ2Rn0kTI8XM7p6urirbfe4s6dOyiVSrnZ09TUFGNjY7hcLpKSkoiJicFms6FSqZiZ\nmWFsbCyoa+IKhYLIyEiefvppqqqqKC8vx2w243K5uHv3Lu+++y4Oh4PZ2VmioqJ45ZVX2L17t3xe\nRWtrK/Pz88TExBAbG0tJSQlGo5Fdu3Zx8+ZNeSo0EAgwOzsblMN7AoHAqsK+sLAwsrKyyMjIoKGh\n4Xu/aX+d+DweDyMjI9TV1dHe3o7T6ZS3gH7e9PS0PEsa6i1wFxYWGB8fl/tXSGdYeL3eoB3Pm5GR\nwZNPPonBYOD+/fvcvHmTmzdv0tbWJncp/b//+z9cLhfV1dVYLBb5cCHpOpLy1tjY2Lq0NP9BJX6D\nwYDVapVPmZLahEqFFxqNRv7/6212dpaJiYmvtY9do9FgsVhIT0+noqKCgwcPkpqaSl9fH9evX+f8\n+fN0dXWt+ehnampqzab2IiMjiYiIwOfz0dLSsqYH/3xbGRkZHD58mNzcXCwWi/ygeP/+fc6dO/ed\nai/WmnRsdENDwxf+udFoxGKxEBERgcFgAGBoaIiOjo6gjfgVCgXx8fFs3ryZI0eOsGXLFiwWCw8e\nPKCxsZHz589z9+5d9Ho9GRkZVFRUcOTIERITE+W21TU1NTidThISEsjPzyclJYWcnBzS0tJITU2l\nvb2dhYUFfD4fjY2NfPDBB+v+Pufn5xkbGyMyMhK1Wo1Go5E7tLW0tAQ98UsWFxcZGhpiaGjoS/+e\nTqdjenp6QxRRrqyssLCwIBcNS/dKaedWMKjVanw+H1euXOH27dtcvXqVW7duyd9DlUpFa2srJSUl\n8mc8ODjI0NCQPHhzOp20tbUxMDCwLu9jwyd+Kamr1WoSEhLYtGkTTz75JHl5eaSnp8tdwDweDy6X\ni/b29nXv3a9QKBgbG/tahy2o1Wp5WeLFF1/kySefJDU1ldbWVt58801+/vOfr1PUa8Pn88lLA8FW\nWlrK3/7t38ptiaVOgh9//DG/+MUvQibpfx3Sdf/ZiviOjg5u3boVtGUVrVbL5s2befXVV6msrESr\n1dLd3c3Jkyc5f/48Dx48QKFQkJOTw9GjR3nxxReJiIiQTw/8n//5H65du8bY2Bhms5mSkhIqKyup\nrKwkLy+Pffv28cwzzwAPH6R/85vfBCXxj46O0tTUxLZt21YdC1tWVkZHRwdXr15d95gkUrX+1xkU\nSEsy2dnZlJSUhEyNwpf57FkDgUBA7h0SzNiHh4c5e/Ysly9fpr+/n9nZ2VV/LrVZt9vtWCwWFAoF\n169f59SP410+AAALc0lEQVSpU3R1dcldXMfGxtatmHLDJ36LxSJvzSotLSU7O5uYmBjCw8MJCwuT\nDwbxeDxyRX8w1rASEhLIzMz8ykKUzZs3U11dzVNPPSXvgb937x7/+Z//SV1d3TpF+8P02V4E8HDa\n8MqVKzQ3N7O8vLwhRjySxcVF7ty5w1NPPRXsUICHs1RPPvkkBw4cYOvWrWi1Wm7cuMHZs2e5dOkS\nJpOJF154gc2bN5OdnS1XM1+9epW6ujpu3Lghb7n1er3Mzs5y9+5dent7uXTpEnv37pWPjAX4/e9/\nz0cffRSU9zozM8PQ0FDQd6h8llQbIZ0B0t7ejsfj+dJr2mKxUFFRweHDh9mzZ8+XVv+Hiv7+fq5e\nvcqJEyewWq3ExcVx/PhxXn/99aAVIkpFwdPT048MHqQZia1bt8o7cxYXF2lsbKSmpobFxcVVhyet\nVw3Lhkz8Go0Go9EoV+2XlpayZcsW0tLSiIqKks8h9/l8LCws0NXVRUtLC42NjfT09AQlZpPJRFZW\nFs899xxtbW2Mj4+zsrJCTEwMiYmJJCQkYDQaKS4uprS0lJycHJaWlujs7OTy5cvcvHnzK6fsQpFU\n3S1NRweDSqVix44dlJWVrWoWs7S0JPeOD4WisW9C2pYYKjdrjUbD7t272b59O1FRUfJBWikpKVRV\nVZGcnExWVha5ublERUXJW1tPnjxJfX09vb29q2pWPB6PvPQkHXfb3t4u12Z8+umntLe3B+W99vX1\ncfXqVSorKzGZTPIoOy4ujuTkZCIiIpibm1u3JZfo6GhSU1PJy8ujuLh41TTy52d/pKrx6OhoCgsL\n5Z4K0iBDKtQNxfNO4OHDusPhoK+vj9jYWPm6+Pwoez3NzMw8tsOn3W5n69at7Ny5k5SUFKanpzlz\n5gzXr18P6imlGybxS0+0Wq2WqKgoMjIyqKqqYteuXZSWlmIwGOQL1+fzydvNhoeHuXDhApcuXeLG\njRtBucFLU1Kpqam89tprXLp0iZaWFhYWFigsLKS8vJyysjKio6PR6/X4fD55O86nn37K2bNnVx3q\nsFFI09ERERGYTKagxKBSqTCZTBw9epS9e/fKrWGlKvmOjo6gdOj7rvR6PYWFhSQkJAQ7FODh5/zE\nE0+Qnp4uV73n5OSQmpoqN3UC5ALFhoYGzp49y6lTp3A6nV+aZKTZjWAdLfx5fX191NbW8ud//ufE\nxsbKD7VSPUJ0dDTLy8vrlvhzc3N59tlnOXToEAkJCXz66afU1dUxMTHxyAhUpVJhs9nkA532799P\nZGQkCoVC3i46Pj7O9PR0yOzE+TyPx8PAwADZ2dk4HA7efvttXC5XsMN6hEqlIi0tjRdeeIEtW7Zg\nMploaWnhv//7v9elO9+X2TCJX6fTYbVaSU9Pp6qqisOHDxMbG4vFYiEsLEweyXm9XpxOJ3V1dVy6\ndIlPP/2UyclJ5ubmgj6qk6Z8MjMz5fOtbTYbJpOJsLAw+RCT4eFh2tra+PWvf019fT1zc3Mh3UHr\ncaSb+ee3P60nqZJfuiFLIxqpNa/D4QiZA52+CZVKRXR0dNAeqL7I0tISHo8HnU6HUqkkPDwcg8GA\nQqFgZGSE3t5eWltbuXHjBq2trQwMDOByuUJyZPlVfD6fXKz72VmXYLyXvXv3cvToURITE9FoNGRk\nZPDqq6/K95jP0uv1cmvzmJgYzGYzSqWSpaUlxsfHqaur4/z581y9ejWkTtF8HGl2IhSX6axWK7m5\nuTz55JNERETQ29vLtWvXGBwcDHpjpJBO/CqVCr1eT1VVFQUFBfIWn6ysLPLy8uQbjNS9r6uri6am\nJlpaWmhtbaWzs5OBgYGgrcVNTU3R1dXF5OQkZrMZvV4v1x5Ie96lbXOLi4v09fVx+fJlbty4wcTE\nBA0NDUGdDloLCoWChIQEoqKigvLzpe0+VqsVvV4vn1F+584d3nrrLRwOR9D3vX8bXq+X4eFheaQT\n7O5ly8vLvP322/T29pKbm4vdbpd7VzQ3NzM6Oir/GhgY+NLtZRuB2+3mxo0bxMfHB+3alpjNZrkt\nLzzsGLpnz54vTIhqtRq73U54eLjcH2JiYoLW1lZOnz5NS0sL3d3dzMzMhPwDmVKpxGq1UlpaSmNj\nY8jNih44cIAjR44QFRXFxMQEtbW1vPfee0xNTQX9ug+5xK9WqwkPD8dutxMVFUVERAQvv/wy5eXl\n2Gw29Hr9qj2Q8/Pz8oV77do1rl69SlNTU0iM8KempmhububatWssLy+TnJwsbwFSq9X4/X5mZmYY\nHx9nYGCAhoYGTp06xfXr14Ma91pSKpXExMRgtVqD8vOlQ2+kXuXSNH9nZydXrlwJSkxrYWVlhY6O\nDkZGRvD5fPJBSWFhYet+oiA8nH79+OOPaW9vp7i4mOTkZDweD/39/dTV1TE7O8vy8nLQv5Nrxe12\nc/PmTcrKyigqKlpV3b/eHA4Hvb29AHL74y+aCZKaa0k9ISYnJxkZGaGjo4Nr167xm9/8Ru6KF8qk\no6jDwsLwer0kJyfT3d0dcom/sLCQoqIiVCoVnZ2dfPrpp1y7di3YYQEhmPhNJhOFhYUcOXKEHTt2\nkJiYiMViQa/XPzJl7PP56O3t5fLly7z55pv09PTgdrvlizvY3G43Dx484F//9V+prq7mwIED7Nu3\nTx7lS3uRz5w5w/nz5+nr69sQ02tfVzCn+B8n1OL5tlZWVujr68PhcLCwsIDRaCQiIoK4uDgmJibW\n/eYtFUu2tbXR2dn5hYViofCdXCsrKyv09vbKn7/ZbA7atXXmzBncbjdHjhyhoKAAu93+hX9vZWUF\nt9uN1+tlaGiIhoYGTp8+TVNTE8PDwywvL2+IfyOdTkdBQQHR0dGMjo7S3d0dkvfN1tZWWltbqaio\noKGhIWiF5V8k5BK/1Wpl586d8rYf6cQ3qZWptE/S7XZTV1dHW1sbfX19dHd3y40dQoU0rTwyMsL5\n8+fldXupHsHv9zMxMcHQ0JB8Awml+L+L7u5uuru7SU1NDWoc8/PzdHd3Mzw8vKqvww+B3++nra2N\nmpoa9uzZw5YtW5iammJpaYn+/v516QD2WVJhbaiPGNeC1+tlfHycN998k08++UR+mB8eHmZkZGRd\nC+MGBgbkpl5VVVVUVFRQVlYm1wyNjIzQ09NDa2sr7e3tzM3NyYd8DQ0N4XK5NlQPC+mzl46y/TpN\n0YKhpqaGvr4+oqOj6ejoYGBgINghyUIu8a+srDA+Ps7t27cZHBxc9fsLCwv09fWtSvwOhyNkq0/h\n/7fq7enpCaknvu9bU1MTJpOJyclJent7g/bepWvm3LlzDA8PYzKZWFpaCnpV7Vp58OABly9fJj8/\nXz76c2JiggsXLtDe3v5HkYSDwe/3s7CwEBK7DWZnZ5mfn6evr0/uMdDZ2flI4m9vb6ezs5P5+fkN\nfV3Mzc1x5swZWltbGRsbY3JyMiRzgHTSp/CowAb4tZHjF7GHXvxr+nP0en2gtLQ08MEHHwSGhoYC\nU1NTgTt37gSOHz8eMBgMf4yfvYhdxC9i/wrBPc9QEITvZGVlhZ6eHv7lX/6FX//61/T09JCcnExq\naio2m+0HU9MgCMLaCbmpfkEQvj6/38/s7Cz37t0DHq73pqSk0NDQsCH7EwiC8P0L9nDgW01TrLMv\n+4xCPX4Re/A8Lv6NHDuEfvwi9uDZyPH/UGMXBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQ\nBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQ\nBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQ\nBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEEQ1s3/A5KuJ9rTT5RuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4c987e4790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label: [5 0 4 1 9 2 1 3 1 4]\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "for i in range(10):\n",
    "    plt.subplot(1,10,i+1)\n",
    "    plt.imshow(train_img[i], cmap='Greys_r')\n",
    "    plt.axis('off')\n",
    "plt.show()\n",
    "print('label: %s' % (train_lbl[0:10],))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we create data iterators for MXNet. The data iterator, which is similar the iterator, returns a batch of data in each `next()` call. A batch contains several images with its according labels. These images are stored in a 4-D matrix with shape `(batch_size, num_channels, width, height)`. For the MNIST dataset, there is only one color channel, and both width and height are 28. In addition, we often shuffle the images used for training, which accelerates the training progress."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "\n",
    "def to4d(img):\n",
    "    return img.reshape(img.shape[0], 1, 28, 28).astype(np.float32)/255\n",
    "\n",
    "batch_size = 100\n",
    "train_iter = mx.io.NDArrayIter(to4d(train_img), train_lbl, batch_size, shuffle=True)\n",
    "val_iter = mx.io.NDArrayIter(to4d(val_img), val_lbl, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multilayer Perceptron\n",
    "\n",
    "A multilayer perceptron contains several fully-connected layers. A fully-connected layer, with an *n x m* input matrix *X* outputs a matrix *Y* with size *n x k*, where *k* is often called as the hidden size. This layer has two parameters, the *m x k* weight matrix *W* and the *m x 1* bias vector *b*. It compute the outputs with\n",
    "\n",
    "$$Y = W X + b.$$\n",
    "\n",
    "The output of a fully-connected layer is often feed into an activation layer, which performs element-wise operations. Two common options are the sigmoid function, or the rectifier (or \"relu\") function, which outputs the max of 0 and the input.\n",
    "\n",
    "The last fully-connected layer often has the hidden size equals to the number of classes in the dataset. Then we stack a softmax layer, which map the input into a probability score. Again assume the input *X* has size *n x m*:\n",
    "\n",
    "$$ \\left[\\frac{\\exp(x_{i1})}{\\sum_{j=1}^m \\exp(x_{ij})},\\ldots, \\frac{\\exp(x_{im})}{\\sum_{j=1}^m \\exp(x_{ij})}\\right] $$\n",
    "\n",
    "Defining the multilayer perceptron in MXNet is straightforward, which has shown as following."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.36.0 (20140111.2315)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"214pt\" height=\"836pt\"\n",
       " viewBox=\"0.00 0.00 214.00 836.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 832)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-832 210,-832 210,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"47\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-25.3\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- flatten0 -->\n",
       "<g id=\"node2\" class=\"node\"><title>flatten0</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"94,-168 -7.10543e-15,-168 -7.10543e-15,-110 94,-110 94,-168\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-135.3\" font-family=\"Times,serif\" font-size=\"14.00\">Flatten</text>\n",
       "</g>\n",
       "<!-- flatten0&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>flatten0&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-99.8131C47,-86.1516 47,-71.0092 47,-58.3283\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-109.906 42.5001,-99.9062 47,-104.906 47.0001,-99.9062 47.0001,-99.9062 47.0001,-99.9062 47,-104.906 51.5001,-99.9062 47,-109.906 47,-109.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"71\" y=\"-80.3\" font-family=\"Times,serif\" font-size=\"14.00\">1x28x28</text>\n",
       "</g>\n",
       "<!-- fc1 -->\n",
       "<g id=\"node3\" class=\"node\"><title>fc1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-278 -7.10543e-15,-278 -7.10543e-15,-220 94,-220 94,-278\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-252.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-237.8\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc1&#45;&gt;flatten0 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>fc1&#45;&gt;flatten0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-209.813C47,-196.152 47,-181.009 47,-168.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-219.906 42.5001,-209.906 47,-214.906 47.0001,-209.906 47.0001,-209.906 47.0001,-209.906 47,-214.906 51.5001,-209.906 47,-219.906 47,-219.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-190.3\" font-family=\"Times,serif\" font-size=\"14.00\">784</text>\n",
       "</g>\n",
       "<!-- relu1 -->\n",
       "<g id=\"node4\" class=\"node\"><title>relu1</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-388 -7.10543e-15,-388 -7.10543e-15,-330 94,-330 94,-388\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-362.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-347.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- relu1&#45;&gt;fc1 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>relu1&#45;&gt;fc1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-319.813C47,-306.152 47,-291.009 47,-278.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-329.906 42.5001,-319.906 47,-324.906 47.0001,-319.906 47.0001,-319.906 47.0001,-319.906 47,-324.906 51.5001,-319.906 47,-329.906 47,-329.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-300.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- fc2 -->\n",
       "<g id=\"node5\" class=\"node\"><title>fc2</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-498 -7.10543e-15,-498 -7.10543e-15,-440 94,-440 94,-498\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-472.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-457.8\" font-family=\"Times,serif\" font-size=\"14.00\">64</text>\n",
       "</g>\n",
       "<!-- fc2&#45;&gt;relu1 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>fc2&#45;&gt;relu1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-429.813C47,-416.152 47,-401.009 47,-388.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-439.906 42.5001,-429.906 47,-434.906 47.0001,-429.906 47.0001,-429.906 47.0001,-429.906 47,-434.906 51.5001,-429.906 47,-439.906 47,-439.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-410.3\" font-family=\"Times,serif\" font-size=\"14.00\">128</text>\n",
       "</g>\n",
       "<!-- relu2 -->\n",
       "<g id=\"node6\" class=\"node\"><title>relu2</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-608 -7.10543e-15,-608 -7.10543e-15,-550 94,-550 94,-608\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-582.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-567.8\" font-family=\"Times,serif\" font-size=\"14.00\">relu</text>\n",
       "</g>\n",
       "<!-- relu2&#45;&gt;fc2 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>relu2&#45;&gt;fc2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-539.813C47,-526.152 47,-511.009 47,-498.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-549.906 42.5001,-539.906 47,-544.906 47.0001,-539.906 47.0001,-539.906 47.0001,-539.906 47,-544.906 51.5001,-539.906 47,-549.906 47,-549.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"54\" y=\"-520.3\" font-family=\"Times,serif\" font-size=\"14.00\">64</text>\n",
       "</g>\n",
       "<!-- fc3 -->\n",
       "<g id=\"node7\" class=\"node\"><title>fc3</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-718 -7.10543e-15,-718 -7.10543e-15,-660 94,-660 94,-718\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-692.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-677.8\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- fc3&#45;&gt;relu2 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>fc3&#45;&gt;relu2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-649.813C47,-636.152 47,-621.009 47,-608.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-659.906 42.5001,-649.906 47,-654.906 47.0001,-649.906 47.0001,-649.906 47.0001,-649.906 47,-654.906 51.5001,-649.906 47,-659.906 47,-659.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"54\" y=\"-630.3\" font-family=\"Times,serif\" font-size=\"14.00\">64</text>\n",
       "</g>\n",
       "<!-- softmax_label -->\n",
       "<g id=\"node8\" class=\"node\"><title>softmax_label</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"159\" cy=\"-689\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-685.3\" font-family=\"Times,serif\" font-size=\"14.00\">softmax_label</text>\n",
       "</g>\n",
       "<!-- softmax -->\n",
       "<g id=\"node9\" class=\"node\"><title>softmax</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"170,-828 76,-828 76,-770 170,-770 170,-828\"/>\n",
       "<text text-anchor=\"middle\" x=\"123\" y=\"-795.3\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;fc3 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>softmax&#45;&gt;fc3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M97.2538,-761.413C87.3525,-747.343 76.2083,-731.506 66.9347,-718.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.23,-769.906 93.7952,-764.318 100.353,-765.817 97.4753,-761.728 97.4753,-761.728 97.4753,-761.728 100.353,-765.817 101.155,-759.138 103.23,-769.906 103.23,-769.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"97\" y=\"-740.3\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;softmax_label -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>softmax&#45;&gt;softmax_label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M135.648,-760.057C140.313,-746.062 145.505,-730.485 149.807,-717.579\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"132.365,-769.906 131.258,-758.996 133.946,-765.163 135.527,-760.419 135.527,-760.419 135.527,-760.419 133.946,-765.163 139.796,-761.842 132.365,-769.906 132.365,-769.906\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f4c045d4350>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a place holder variable for the input data\n",
    "data = mx.sym.Variable('data')\n",
    "# Flatten the data from 4-D shape (batch_size, num_channel, width, height) \n",
    "# into 2-D (batch_size, num_channel*width*height)\n",
    "data = mx.sym.Flatten(data=data)\n",
    "\n",
    "# The first fully-connected layer\n",
    "fc1  = mx.sym.FullyConnected(data=data, name='fc1', num_hidden=128)\n",
    "# Apply relu to the output of the first fully-connnected layer\n",
    "act1 = mx.sym.Activation(data=fc1, name='relu1', act_type=\"relu\")\n",
    "\n",
    "# The second fully-connected layer and the according activation function\n",
    "fc2  = mx.sym.FullyConnected(data=act1, name='fc2', num_hidden = 64)\n",
    "act2 = mx.sym.Activation(data=fc2, name='relu2', act_type=\"relu\")\n",
    "\n",
    "# The thrid fully-connected layer, note that the hidden size should be 10, which is the number of unique digits\n",
    "fc3  = mx.sym.FullyConnected(data=act2, name='fc3', num_hidden=10)\n",
    "# The softmax and loss layer\n",
    "mlp  = mx.sym.SoftmaxOutput(data=fc3, name='softmax')\n",
    "\n",
    "# We visualize the network structure with output size (the batch_size is ignored.)\n",
    "shape = {\"data\" : (batch_size, 1, 28, 28)}\n",
    "mx.viz.plot_network(symbol=mlp, shape=shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now both the network definition and data iterators are ready. We can start training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:\u001b[91m[Deprecation Warning] mxnet.model.FeedForward has been deprecated. Please use mxnet.mod.Module instead.\u001b[0m\n",
      "INFO:root:Start training with [cpu(0)]\n",
      "INFO:root:Epoch[0] Batch [200]\tSpeed: 21765.89 samples/sec\tTrain-accuracy=0.111300\n",
      "INFO:root:Epoch[0] Batch [400]\tSpeed: 21178.05 samples/sec\tTrain-accuracy=0.113800\n",
      "INFO:root:Epoch[0] Batch [600]\tSpeed: 22749.74 samples/sec\tTrain-accuracy=0.145050\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=2.770\n",
      "INFO:root:Epoch[0] Validation-accuracy=0.245700\n",
      "INFO:root:Epoch[1] Batch [200]\tSpeed: 22469.74 samples/sec\tTrain-accuracy=0.401450\n",
      "INFO:root:Epoch[1] Batch [400]\tSpeed: 22132.54 samples/sec\tTrain-accuracy=0.752800\n",
      "INFO:root:Epoch[1] Batch [600]\tSpeed: 22298.21 samples/sec\tTrain-accuracy=0.830450\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=2.697\n",
      "INFO:root:Epoch[1] Validation-accuracy=0.851800\n",
      "INFO:root:Epoch[2] Batch [200]\tSpeed: 22641.03 samples/sec\tTrain-accuracy=0.863100\n",
      "INFO:root:Epoch[2] Batch [400]\tSpeed: 22535.41 samples/sec\tTrain-accuracy=0.889600\n",
      "INFO:root:Epoch[2] Batch [600]\tSpeed: 20554.33 samples/sec\tTrain-accuracy=0.904700\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=2.751\n",
      "INFO:root:Epoch[2] Validation-accuracy=0.910900\n",
      "INFO:root:Epoch[3] Batch [200]\tSpeed: 22567.61 samples/sec\tTrain-accuracy=0.919100\n",
      "INFO:root:Epoch[3] Batch [400]\tSpeed: 21676.39 samples/sec\tTrain-accuracy=0.926950\n",
      "INFO:root:Epoch[3] Batch [600]\tSpeed: 22239.15 samples/sec\tTrain-accuracy=0.935050\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=2.715\n",
      "INFO:root:Epoch[3] Validation-accuracy=0.937000\n",
      "INFO:root:Epoch[4] Batch [200]\tSpeed: 22807.12 samples/sec\tTrain-accuracy=0.940900\n",
      "INFO:root:Epoch[4] Batch [400]\tSpeed: 22676.59 samples/sec\tTrain-accuracy=0.946550\n",
      "INFO:root:Epoch[4] Batch [600]\tSpeed: 22396.59 samples/sec\tTrain-accuracy=0.949450\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=2.659\n",
      "INFO:root:Epoch[4] Validation-accuracy=0.948200\n",
      "INFO:root:Epoch[5] Batch [200]\tSpeed: 21642.09 samples/sec\tTrain-accuracy=0.954500\n",
      "INFO:root:Epoch[5] Batch [400]\tSpeed: 22713.46 samples/sec\tTrain-accuracy=0.957200\n",
      "INFO:root:Epoch[5] Batch [600]\tSpeed: 22707.05 samples/sec\tTrain-accuracy=0.958150\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=2.692\n",
      "INFO:root:Epoch[5] Validation-accuracy=0.957600\n",
      "INFO:root:Epoch[6] Batch [200]\tSpeed: 22707.33 samples/sec\tTrain-accuracy=0.961650\n",
      "INFO:root:Epoch[6] Batch [400]\tSpeed: 21229.35 samples/sec\tTrain-accuracy=0.963350\n",
      "INFO:root:Epoch[6] Batch [600]\tSpeed: 21281.46 samples/sec\tTrain-accuracy=0.964600\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=2.770\n",
      "INFO:root:Epoch[6] Validation-accuracy=0.962200\n",
      "INFO:root:Epoch[7] Batch [200]\tSpeed: 22256.75 samples/sec\tTrain-accuracy=0.966750\n",
      "INFO:root:Epoch[7] Batch [400]\tSpeed: 22260.44 samples/sec\tTrain-accuracy=0.969000\n",
      "INFO:root:Epoch[7] Batch [600]\tSpeed: 22725.88 samples/sec\tTrain-accuracy=0.970050\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=2.684\n",
      "INFO:root:Epoch[7] Validation-accuracy=0.964600\n",
      "INFO:root:Epoch[8] Batch [200]\tSpeed: 22429.85 samples/sec\tTrain-accuracy=0.971950\n",
      "INFO:root:Epoch[8] Batch [400]\tSpeed: 22352.06 samples/sec\tTrain-accuracy=0.972100\n",
      "INFO:root:Epoch[8] Batch [600]\tSpeed: 20139.16 samples/sec\tTrain-accuracy=0.974400\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=2.786\n",
      "INFO:root:Epoch[8] Validation-accuracy=0.966900\n",
      "INFO:root:Epoch[9] Batch [200]\tSpeed: 21898.75 samples/sec\tTrain-accuracy=0.975250\n",
      "INFO:root:Epoch[9] Batch [400]\tSpeed: 22031.53 samples/sec\tTrain-accuracy=0.974700\n",
      "INFO:root:Epoch[9] Batch [600]\tSpeed: 22043.60 samples/sec\tTrain-accuracy=0.977300\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=2.736\n",
      "INFO:root:Epoch[9] Validation-accuracy=0.968200\n"
     ]
    }
   ],
   "source": [
    "# @@@ AUTOTEST_OUTPUT_IGNORED_CELL\n",
    "import logging\n",
    "logging.getLogger().setLevel(logging.DEBUG)\n",
    "\n",
    "model = mx.model.FeedForward(\n",
    "    symbol = mlp,       # network structure\n",
    "    num_epoch = 10,     # number of data passes for training \n",
    "    learning_rate = 0.1 # learning rate of SGD \n",
    ")\n",
    "model.fit(\n",
    "    X=train_iter,       # training data\n",
    "    eval_data=val_iter, # validation data\n",
    "    batch_end_callback = mx.callback.Speedometer(batch_size, 200) # output progress for each 200 data batches\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "After training is done, we can predict a single image. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAAFfCAYAAADptc+BAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnWtXIkmzRhO8gZeeXvP//+NMtwqCIOfDe4J5CCIyExss\nxb3XqlVF2Q22rZswMjKiFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+CaMBn79zcCvDwBw\nCppuHX/EZwEAALsgXwCAAUC+AAADgHwBAAYA+QIADADyBQAYAOQLADAAyBcAYACQLwDAACBfAIAB\nQL4AAAOAfAEABgD5AgAMAPIFABgA5AsAMADIFwBgAJAvAMAAIF8AgAFAvgAAA4B8AQAGAPkCAAwA\n8gUAGADkCwAwAMgXAGAAkC8AwAAgXwCAAUC+AAADgHwBAAYA+QIADADyBQAYAOQLADAAyBcAYACQ\nLwDAACBfAIABQL4AAAOAfAEABgD5AgAMAPIFABgA5AsAMADIFwBgAJAvAMAAIF8AgAFAvgAAA4B8\nAQAGAPkCAAwA8gUAGADkCwAwAMgXAGAAkC8AwAAgXwCAAUC+AAADgHwBAAYA+QIADADyBQAYAOQL\nADAAyBcAYACQLwDAACBfAIABQL4AAAOAfAEABgD5AgAMAPIFABgA5AsAMADIFwBgAJAvAMAAIF8A\ngAFAvgAAA4B8AQAGAPkCAAwA8gUAGIDLoT+B78poNHrXvdbjFpvN5qA/3/r7rec79PX+9PMD+Cog\n3wEYj8dlNBq9+8j+fil9cjxUmJvN5qAjep7ounYv+7sA5wLy/WC8QMfjcXq0Pu6PHqn56+ixv//2\n9rY9NpvNzmP/MRWwv47uZY9Ho9GezAHOCeQ7ACrVi4uLcnFxkV7X7vmP98hOz7Vrfbxer8vb21tZ\nr9fN67e3t53X/ZMj+pwAzgXk+8Fo5HtxcVEuLy+bZztqjy8uLlJ5ZULrTQusVquyXq+7zv71fETs\n70XXKnCLgAHODeQ7ACrfi4uLcnV1tZVodH11dZVe29nkW0p/1Gm0pLxarcrr62tZrVbNay/R6Kyp\ni/F4vBMx6z0TLwKGcwT5DoCmHFS019fXW7naYfdq5+vr6zDyzSLPWhQcnV9fX8vr62tZLpfba38s\nl8sdkapgfZ7YP9YFw1LKzj3EC+cK8v1gorSDSjQ6bm5uqtc3Nzfl8vKy+9f6moCje8vlsnpYzlnl\n23Os1+u0VM4iXoBzBfkOgE87qIBVqDc3N2UymWyva4+vrq7SX/Wje6XElQfR9WKx2B7L5XJ7bblm\nk+9oNNpZeDPBRtcW3a7X652vjb22iRcBw7mCfAcginxVvibV7JhOp3uPr66uDvp1v5R6qZc+fnl5\nKYvFory8vJSXl5dtrtmLt5SyVwVxcXGxfTwej3c+FqFROuKFcwb5fjCadhiPx9ucr6YSVLDT6bR5\n3N7eluvr67T2NrtfSrw45+/P5/OtdGvitcU5LT9br9fbdIT9my3doHL10vWbRwDODeQ7AF68PupV\nqfpzdH13d5fKt2dDRK00bbPZbN8YVL4mYN0QMhqNtmVnPcdqtdrKeDwel9VqtbeDz7BoHT6GaJGz\ntfDJwuhhIN8PxqSiZWYa8ZpU7+7uyu3t7V70a6kGXXgzGfbuQqvtRovuaaTsN4j4Nw+Tr49+s3tW\npubrhf01P9h/TkuoPYu0+lj/fm3NAGKQ7wBk4jLB3t3dbY8o52sRskWjKt8s5+uve3K9eu2rNPSN\nQ6N2law/+3uHbt6A9+EFm92rLZD660zI2W9UsA/y/WB8vtdHvpZSuL+/L3d3d3tVDT7q1VRAa1ND\nLeq1cyZkv0ioeerr6+uyXC7LZDLp+gH2Qvayje7xQ/w+ov9jO/t7ta+/v9fzRm9v2vzfxSDfAbC0\nQyRfSznc3d2V+/v7najSl6JlkW/Pr4+1H0L/Md+HwoS/XC7L9fV1mUwm25rfrMwsq/ONfrijx/wA\nv4/e1JItluquRX+2a61g0f9j++1IF1T5f8tBvgPg0w4a3Wrka/JtbcIwAff8CliLerOzz/GuVqty\nfX0d7nQ7pOIiW4SL7vND/D56vxc2m/2djPaGqvfse8H+n2zBVDfMaKoKcpDvB5PtblPxWtR7f3+/\nt93YbzvWyPdPpFu7p5+rRUA+IrLjkLxzb1UE8n0/tTdj/zHbQOPPtqFmsVjs5P5NwFrp4p8fcpDv\nAKiAs5yvCThrtuOvvXxLaW+iMCIB67VFu7UIVascsjyzP9eqIJDvcehJQ9m1baTRDTVaVqjlf74v\ntb2Wndkk0wb5fjBZ5Kvy1cjXt42MWkleXl6W8fh/4/ha+Vu9Z0Ri03vr9bpcXV01F8+yRb3onk89\ntErTkO/76HkTtPPLy0uZz+dlPp/vbahR8Wp+16caLE3FBpk2yHcAoqqBqM7X5Ftrpm73Tb6lHD7J\nonU/WsVulbD1HJl8o2vk+z58vj37v3t7eysvLy97C7m6iaaU/8rRotfRiNe+H1l0y0G+H4xussjq\nfHXBrTY2yO8w87Qi2h6ilEVvWqP22Ee/WS2wRtRwOD0Ln3bMZrMd8er2cX2z9HXX/v+SyLcP5DsA\nKk/79U63Fmv067fbRtdR3u1Pee8PTi3t4c9ZCgP5Ho+abP1h3fGinh0qXpWv/23IytCQbxvkOwD6\njWzlPJZve35+3m4jjgZt1q7fk26IWjdG173nHjKR+gnNev8YUfxXksGx/m369ayJd7PZ7JU03tzc\n7FS0aD6+lLL3G5d9X3t5Qwzy/WB8FPH6+rpdXZ7P52U2m5Wnp6dyc3NTSinVtIOfdBz9iq/X+tiv\nWvc87jmMmsDt2j4f/xwqXv9n3yNhL4HPKIXav6H2sejf4r/OOpapNm3Edipqn45MvtFim1a+kHpo\ng3wHwEe+Jt/ZbLazfbiUfflqnlcfq3xblQYmX/0ByY5amiO7VxO33Te8gLX9pN23etEsio+uD3kj\nGJpW9UntN4XoWh/r17NV7WCRr26amUwmYemfoqmi1WpF5NsJ8v1gNPK1b3CNfJ+fn7edyqxsx1c1\nRGddFPHSjUrAWhL1aY3ocfaxTN5RVKuoMOxz1HrRVh5Zr2vplM8g4J6Kkz9JGenXzL43VMTRG7PK\nN+o2pzl5e25dKF2tVnsLdZCDfAcginzn8/lOmc/FxcV2ASMrN9PHWU4vWuG2H8SaUP19jbhr1/7v\n2Q+7/jD6HHX2q7OKo5S+PhQ9uekh5duTXmjl7kvpf3Ox/2v7PlDx+sPEe319HaYaVL5+wdRSFMi3\nH+T7wWh+zOd8Z7PZzjevLV7o5IjsMNFFzWyie1EKo5bWqL0J+EjcX9u/OcpNK14aPqfYe63P18pl\nfxStvPQhVSLRG0h2T+Waidee13p21MSrEbOvgNAp1uR82yDfAdC0g1Y7+B1FJl8v4OjaZB2VaUW7\nxVppjEiy/k2g9qagUbuKx34gW5FclvfMxOHz2fpaLQl/BD3ybUX1dp29iUT/tuzrFD1W+WbRrspX\nAwiLev2EE8hBvh+MfeNatGBpB/vG1ehQc2jZtmIv355+CT6dUdtBV3vN6Np+UE26m83/GvMYkRDt\nvhLJuCVd/Vi02DdU9HtI1NtzXXtD8fdaItezyfT6+npPvCpf/bOWplgulztvxkS9bZDvBxNFDYvF\nYuebtpT/RccaTUSHSs/Lt9YnN0pn1KJa38BHG/us1+sd6fq+woYuuLWi3tavz61DX6/nOPX/d8/H\neqNTL9+ahP1vGNnrGev1eiveaJ3Ay1d/c/O9IEg7tEG+A+DTDtk2Tosmsi5mes/kG80/89cmzN6o\nVn+w9Nqex8bWq4Az+eqij93Xj+u1l4g/ohK6TL61srpTkwnYy7AW0ev997yxRJ+Dv9eKdo3RaLSz\nQWixWOyJlwW3Nsj3g/GRry1QaMRrH9Nv6uxs1+PxeK+3bnaYNGuHf3477Dks6r26ugpFYaj8vCyj\nVEN0+IUiex6Tuf9YS7xRTfIp/7977rcief91y75Of/LG0op2DZXvYrHYa+pPtUMfyHcAVMCafvC/\nqllu+PLysry+vparq6s0FdEb+VrDc41ua5GvF3DruidK1885qhGOzirXqIROH6uMaq/xkQtCkWyV\nKIqP3lg2m0213NBfK70y9K+plQ1Z6VktUoYY5DsA+o0d7QpS+aoQ/Yqyimw0GjXzvdrwvLeKIsr5\n9og1ErvVL0er4q1Dv15RPbMKwos3K6v7qOislXe1/+vozSU6134D8qmoWuoh+rfXxKvfr7Xa3yxi\nhl2Q7wfjv7HtG/j19XVHvPZnTMwtUap8eyoeWnXDUbVDJP3eCDr6c1mpW3TP/7ZQK6UbjUbdZXQf\nmXaIFrnsOnsz8RJ8e3vbGaCqhy2YlVK2Ub3Pu/vPzcs5irqjyFffyHtSFbAP8h0AFYnlar14N5vN\njiRrpWAmkUhEmbAy+WXXPRGyL4mrnWtH9G/NfvW1NIreK6V0v7GcMvVQk60/926OeXt7K5PJZHtY\nE36r3y7lf+K1Rc9S9munvXCzCLgn5VCrBUbAdZDvAPi0g/9hsPv+V/NaNGfyzX5w/ePabrbouvUm\noPcjwWaS7n1sb0a+30DUf6CU0iV/+9X8lP/PvedWNK/3rN+z9X6OxKuLkpGEs4g/Sjv4z6Un7QBt\nkO8AZOItZb9DVCtnqbnL7NfU6FfD6O/Xnr/2JhBFz4emNFrXloLxi4dRJYelHVoVHaeUb020mXx7\n0kZvb/8b92Oj3O1eKf+J1ypRfMpBpRsJOKqsqEW+rbQDEq6DfD8Y/aa2PK1h6QBLN9QE6a+1Rrjn\n7KsJapUGvYLuiZR9hNxzWB2xH1Wvj/V6NBqFC4HR9akrHrxsMxlHEWV2b7FY7FSulLIr3pubm50I\nVMWbSdenvLI3cf9GF0k4+nfCPsh3AFS++ng8Hu+Jt6dWVRdXskUTf/RuQog+hx5R11Ikmkdu1THb\nWeVrvQT0Wu+NRqO9UrioPM7qo0/5/6z/L/7/SB+30ikqu9fX163o7Ouu4rU/q59HJOBMxP7Nuhb5\nZuJFum2Q7wBotKPitR8kPwlAF+OyAnt7Xn3+2g999Fy992pHFLFnueRIjtnx9va2/VVbj+ieydf6\n0+rz+McXF7u1sKf4f+75P4nSJ1laRcVqX1OrgLCvxyGRb/Z5ZwKO+vtG4kXAdZDvB2PfkBa16A9I\nJj27Nvy9qJZTv/FrPwy158o+Fn1edu5JS9gRCTG7VvnqOboejUbbv6u7r7Qsyx5/hHxr0vXy9amV\naJSPfs9cXFxsI97JZFKWy+Xen7M/25PzjX57yqpMetIOkIN8B0C/OXvrTI9Vj+p/IN9D7e/15Irt\ncSbE6PF6vd4KtnWYfLN6WD0uL0/zI1ATbXRoGqV2Xq1W26/z5eXl9t84nU7LYrHYyldz+/Z5tK71\nc/fi7V10i95sIAb5DkzvN+gpvpFP8ZyWQonSKT5H7D+X2g+9j3z9taYf/G8RtUWkU0W+NflG96Pc\nrkWvmqP3aZ0sX19KvoNNf+PS+z614Jvn6Lirl5eXrezta+/TEci3DvKFk+B/yFXGerYf+PF4XF5f\nX/eeQ4UZ5Xh1p5Wd7blNIl5K+vkMkfP1j6M3nFL+mzxsOxjt36W/GWgFh1aUqJD18/Gfn71OKbs9\nev0bm4nX5BsJWFMjpB/aIF84Ol68eh2JN4rWVEoWAfoqB78QpQtEJnQfFXrZDVlqFolYpeUrSey+\nzvrTTmImXi/d6HOKHmvUqwL2Ua+JV+UbvRki3jrIF05CJF7DFhsj+drf9Qs9vsY0qgDQ8qtoE4vK\n7SPk6wVbu2f4BU8vUc1n+2ZGPQLW1/LXmm7QyNdHvbPZbCfyVQEj336QL5yU2g+gydHXpL69/TcR\nw+/2qtW/9shXI2Mr6Tv1vz+TrZeuz99mNda6GBlFvl6+tdfX6yzyzdIOdo+c7/tAvnASoh88XTjS\nPKN+3C+GWd2zyqG24m5C8c+rz+13EZ7631+Trl1n/TqirdtR2iFq0Wn43Hst5aG/UdTEa/L1C57k\nfPtBvnB0TLJ2nf0Z/1irJCxna0IxofqFqagqwiJffW59Xi+4U34deq6tisHK3vRz0t2AdkRpB7+F\nO4p89WsRHb0539lsthPxar7XytzsgBzkC0elVsqU7bAy8dpCXFSa5qNXjXT9Pb9opxUWuoMwKnk7\nxdcje6x58aurq+21/Xs18tVdeT7yzean+f+H1gLfITnfxWKRLoCSdugD+cLJaIlYNwJoTbAedi8T\nRvRYpWsfj+phT53v9f/m7L5WelhpmX+svRv8RpRaqZl/bRVwtn3Yb9+Oqh20uY/fCk3aoQ/kC0en\nN+3gtytHZWdRnWqWs9Q/40vd/HPWNiIckx756JuBNY6PIl/dsddT45t9Hv6NS+XrI1/L6/rod7FY\nhHl3Ftz6Qb5wErIFN6UmvkPqVKOPvfe5h0AjXOtjYfe1/abJtyXg7A0m+u3AyzcTsE87LJfLvXRP\nlBKCHOQLg9Ej0c/43O8hE75POWirTe3bYFMrdIqFjRHKSs4snePlmh0mWL+Bwm+isEPl7a9JObRB\nvgAnJEpv+Osor2tdylS6d3d35e7urtzf35e7u7tye3u7J2AVbym7PRtqE0BWq1V5fn7e20BRKyPL\nGukg3T6QL8CJ8OKNctlRRcP19fV2QKbJ16R7f3+/E/1Op9OtfDX9kMnX9z7WfhmRfK2qwS+m9QoY\nEecgX4ATEi30+Xs+r2vphh75RqmHWuQbNcyx69lstiNfq+WNIt+swoR0Qz/IF+DEePFG5XSadohy\nvSbfh4eHrXRVvjqZw+9wi+Srka0dWdrBImRfyZAJFwn3gXwBTkhNuL7ETDdRZJHvw8PD9mP256JF\nt1bkqxUMVsWQpR2idpHRwhrCPQzkC3Aiog5lUbMcTTto5JvJtzaVo7Xg9vr6Gm4XzuSbtYu0crha\nzTXUQb4AJ6QW+er2ad+3oSbf2jRmL2C/ddhHvpbn1Xxv1Cy91jSHhbb3gXwBTkxLvFHkW6t20CY7\nJlvf2aw37WCR79PTUzXyjVp32vPr2V9DDvIFOAHRpopMwlGpmUW+WuNrka9upIjOtZyvVjlYhDub\nzcrz8/NBdb6+Y1nPjkbYBfkCnIhausEfrU0WKl/NF2fnQyJfE+/z8/NeysF3L4u2DiPZ94F8AU6E\nr2bIDi9cX8WgHcyur693nrt2lBKPZMomVfitxL68zFc5wJ+BfAFOhFUyZPlZXWTLtgv7RTR7Xi9Z\nJaq5zXo89MzEQ7inAfkCnIioWY4/Li8vt7ld3TShW4ajCRVZox5rRZn17fWtH6NevF68CPg0IF+A\nE1DbveYPla8J2Ee+0WDMrCexCrindWQ0iSLbzQbHA/kCnAjN9WoVg8/paueyKPLVXWu1JvC+ib0K\n2Eexvp1kJl6/wIaAjwfyBTgRtR69VserFQ0W9Vrkq20iff2uPb/HC9inHXpyvtEIeCLf44N8AU6E\nn1Dhd675xjkmYIt8s11rLWoLbl68rZxv1MsBjgPyBTgBtd1rUQ1vFvla2iGazdYjwlbagWqH4UC+\nACfCdyzLxGuHNkevRb5+Tp2Xoo98o7RDNDDT53yz9pFwHJAvwImo9ertjXw15+tLzFTCWWObKO2Q\n5X01FVGLfBHwcRi3/wgAvIesXWQU+fZUO2SlZp6oxWPvght1vh8HkS/AgdSGYRom26hNpF9081Gv\npRy8eHU6hZdqdp1NI462Eke1vlQ7nA7kC3AAPT0VRqNRuby83JOtlpbpKKBoFlutyiFKH2TXv3//\nLo+Pj9uWka22kT79wDj404F8ATrxEyhqZ98sJzq03ExFrSOBaj16/c606HiPfGt5XzgeyBfgALJe\nvL5FpMo3krCKdzqdbtMSutDmUw4mX4tso/Hv/vz79+8dAdf69VoXM416ST2cDuQLcAAqXt/UXB/7\nrcS19IPJ1xbkfNrBVzroWCBtCakDL+368fGxGfn6GW3ZFmM4LsgX4ACyHr0qSltky9IOurhm8m3N\nY/OLbX4gpjVG1wW2l5eX8vT0VJ6envYi3yzt4PPJ5HxPB/IF6ERLvDT69f15bbHNN0evpR2i54ha\nSZbyX9rBIl8/EshPJNao1yJfXwFhKYdaiRniPS7IF+AAMvFq5Kry7V1wiyZcRAIuJY98bRabn8nm\nzyZmTVOYfLUe2O+OQ8LHBfkCHECUdlABa8/e1mKbXkeLd36xzXaymRyzeWyWatA0gz9HaYdMtGwv\nPg3IF+AAVLzRlIpMvFnu145Dh2Jmka+NgbcKB5OsCldzw5F87TWiazgeyBfgAFqRr5dvS7x2rc9d\nuy4lz/laasEW2H79+rUj2ehaS80iybYew/tBvgCd+Brflnitblfrd/1EYista6GTKVqj4FXCWoKm\nOV6t7bUSM8T6sSBfgAPIIt8o7aBy9XW7PpVg+HaQ/nqz2eyMfveH9m3woo22DpNSGA7kC3AAfneb\nHwEfRb5Zk5yoRWQr52rbirUPb494fe8GNk8MD/IF6MTX+WaRr4q3FvlmXcpah2+A7rcV6y63qGtZ\n1LEMPh7kC3AAhyy2RX0aot682gy9lLhdpJ4PiXxrrSKR77AgX4ADiHo76JZgX+Xgo9+etEO00cGu\ndaEtE68KOGsVqQKGYUC+AAfgF9x6qh1UvFnawYgE7I8o6vUStsoGlS2R7+cC+QJ00io1i3K+mu9t\nVTvUot5o6GUt7WACrjVbZ8vwsCBfgAMwYfoeDBr5tgQcpR2ygZdemrWoN5Jw1izH55Lh40G+AAdQ\n62hWq3aIIl8/HigaeOn760bTKrJ872KxCEXrr2EYkC/AAUSbLGp9HbKZbF68pdTzvZGAa/leO9ea\n45ByGBbkC9BJNEKo1tHskJxvFO1GUa5KNhob5Gt6tZpBpatnGAbkC+DwEane9ymHrOJBxduztdgP\nxcxyua+vrzt9ebNpFD6XWxMuEh4G5Avw/0TS1Xs91Q6RgFtbi33EW9sqvFgstpMpanPYsh4RtXvw\nsSBfgLIvXi9dO/sNFrVa3yjqzXK+tU5ltnhm19n4d9+7wedzEe7nAvnCtycSbXT93sg3SzvY82q+\nVysYtO+uNkLPRsDrHLZaH14k/DlAvgABKkc7HxL5ttIOh0S+1qNXh2LqIEyf8601zUG8nwfkC/D/\nRML159b8NhWwr3SImuqU8l+lQyZfE69J14+Az+TrIff7uUC+AEImXLv289tqAzR1onGU740iX11w\n05SDHwGfpR2s2sHX80Yg3mFBvgAlX2CLFtt6xHt9fb0T8Wa723zFg498Lbq1yPfx8XEnBRHlfDXy\nRbyfF+QL8P9EC2x6neV9MwGrcGu721pz2XQm29PT084kYh2E6UvNSDN8bpAvgODTAe8V783Nzd7C\nnKYsok0WPUMxHx8f9yYR+1Iz7dOLbD8vyBe+NV6utUNF6pvq+EW3q6urHVH7KodDIl+V7/Pz845w\n/cQKSzvQs+Hzg3zh2+MX0jJpTqfTcnd3V25vb8vt7W2ZTCY7Eyui5jnRnDbfb2Gz2exMnHjveCAa\npH8tkC98a/xUCp+j1WuT793dXZlOp2U6nVanFGvUbOgU4s1mU0ajUbVPrzbO0SMaiJltsIDPCfKF\nb49vD2kpBH/OIt9oUGbWMtLOvszMTyPOJlP4Pg5Evl8X5AvfGs3n+l1qdm3n6XRa7u/vdyLfLO1w\ncXGRVhv4+7UevVHawbeZ9JEv0e/XAPnCt0fTDjqNwjdG95FvlnawyLfWuFwf904k1qoGHYjZ2lIM\nnxPkC98aLR/z8tUFtclkss33mnhbC261BTYd5ZOlHLLpFH60kJ/ThoC/BsgXvj1R2sGEa+kFi3ot\n7aACzsYFmQy1c1k0nTiaVlGLgFW20TXy/RogX/jW+GqHq6urHfFaiuH29jaMfGtz2krZX1zzAu6Z\nyRYNxfRz3rzU4fODfOHbk+V8VcBR1OsF7DuX9Yg3mtXWWnDLhmIylfhrgXzhW6Pbhi8vL3e2B08m\nkx3x3t/f70TCJt5atUOU883kW5vbpgLOBmEyGPNrgXzh2+F7N/gpxNfX1zv53tvb2618NQfcqvMd\nj8dlvV5vXysSry60RTvbovQDnAfIF86SqDdDdN/qd/3imh4+zaDCNWlHW4mjfg2RXP1cNr+Zgp1r\n5wnyhbPDotnWeTwe7+R2Taw1GWvdry2yRbPZMvlGE4lfXl72GqTbx7RTGfI9L5AvnCW+/WN2bZsn\nMtla1Gtn3YARLbJl8lXp+onEKl9tFRnNZIPzAfnC2RHNWouOy8vLnWi3J+3g57NpnjeaTJG1ifQT\niX3aQRvoIN/zBPnCWRJ1K/P9d02+ms9tCdj37LW0g59QkcnXxKsDMK1Xrx8NROR73iBfODuiqRNZ\ntzKf620dkcB70g61Bum1mWyticTwdUG+cJaoeKNRP3btpdsSsU9b+DFBvlVkbSimTaaoTSQm8j1f\nkC+cHVHO17eLtMNkW1tw0y5m0YQLP5utlN3I1yJYXWTzc9nsMWmH7wPyhbOkJl/dleaj3myhzQ6d\nwaala3pdSt9E4tlstp1IbEJmwe37gHzh7PA5X797TQV8SL53Op3ubNbQ1/P3WgtumnZ4enraKUEj\n8v0eIF84S3zkq9UJKt6eSgeLgieTSSml3VNBezq0Ftws8vU9e9lkcf4gX/hy+Nlo/r5fZFPZaioh\nahOpKQldULNoOppEER2t9pDREU0kJuo9X5AvfBmiX/Wj62wqRdaf189l813KfC7Xt27UfrpevlFz\nHFt400NFHYkXAZ8fyBe+HL5Rjj/rIpvmeHUihT808m1NIvZ9eaPm5tFIoFok3DORGM4L5AtfiqhD\nmV77yFcnU1jawaLeh4eHbcQbTaeIplJE44D8OJ+3t7fmFOKeicSMBjpvkC98CVS0+jg6dJFN63n9\nLDZNN2g3M92EkUW+Gu1qikD79Pb05j1kIjHyPS+QL3wpatJV+dqCmy8riyJf+5ies7RDFPlG04Qz\n8WYRcPQcpB3OG+QLXw4v22jjg5aXtSJf3yYySjtEka+mBVojgbKFNztHz0fa4bxBvvCl8DneaIdZ\nFvlqtYMLrxuqAAAPEklEQVRGvlGnsvdEvn4IZjQWKEs71BbviHzPE+QLX4ZsFJDuZosW3LSHQxT5\nZp3KdESQ0pKvilcrGWo1vj63mz2G8wH5wpciqnDw4vWlZq3I1zdd9499k/RD0w6+nCyKfvW5a9dw\nPiBf+JLUot9aqVkU+Wbz3XxaI9rdVks79JSaWdrBg2zPH+QLXwIf0fprPd/f35cfP36Uh4eHcn9/\nv1PDq6VkfvyP70ym1Ho1REMx5/N5eXp6Snv0WikZUe33BfnCl8AW0rLFMT3u7+/Lz58/QwGbeHUh\nTQ+PitHLV7cH+45k1jQnms2mbSKZTvF9Qb7wKfEiVPlqOZiOcrfH9/f35a+//ip//fXXNq9rXcm0\nhtfL16PirUW+1odXh2KqfKPxQBr5wvcE+cKnIutYpiOBogkU+vj+/r48PDxsI9+afK2Swb/uZrPZ\n5nntOhoFr03QLcLVRumtyBf5fl+QL3waMvHaxyzy1S5l2ptBKxlsQc1vIfYbKKII22hFvppy0GGY\n0Wy2KOeLfL83yBc+BS3x+sj35uYmbAtp+V3tVqZdy6Kcb4YXYzaZQiNfa47uh2Ja6oHIFwzkC58a\n3yrSbxe2el09tDevn9EWpR18hKtnvW7NZLORQI+Pj+Xp6WkrXXK+EIF8YXBakyns2stXN0v8+PFj\nu8im+V1/jhbcDC/CaCxQbSSQl6/NazPxMg4eFOQLnxIvZE07+Jzvw8ND+euvv8rPnz/Lz58/y+3t\n7c54eG2aY4fP+dqiml0bvodvz0y2x8fH8vj4mM5ls8gX+X5vkC98OqJFsFrka5sqfv78Wf7+++9y\ne3u73eGWnS3nOx6Pt7W2NenWphFr5Pv8/LyVbzTFgpwvGMgXPhVZ9YG2idScr5WVWeT7999/l+l0\nutObITu0zteIUg++l0Ot2kEjX99oJ5pWgXy/L8gXBqU1j83Ofgux71rm0wwW1WqjHO394JvltPrp\nrtfrrWTtiHK6mmbIJlPQnxdKQb4wINkMtmjLr/ZysCPrRKaS9bL1z6u5XG2O48W5Wq3Ky8vLtnzM\njqiSwdIK0UQKxAsG8oXBiaZR+HuRcGvphJZ0FZ9O0EnC+tia5Wj9rka+tqAWTR+O5IuEvzfIFwal\n1RRdm6Nr5Ou7m/kI2LeHzPo4+IW02py1+Xwebp7QdIPmd6PJxr5ROnxfkC8MRtYUPUoZ9ES+Preb\nRdT62qX8J+CoW5ketcjXl5GpcEk7QATyhUGpiVdFWot6fb5Xe/S2Ug+1bmUm1qxTmeZ7fc5X63hr\nKQck/H1BvjAoUdrBR7FZA/WeErJWvreUvp4NWsvbWnDzO9iidAPSBeQLgxNFv16mWZVDdkRVFP6x\ntoyM6ne1PWTUqSyLfC3nG23SIOoFA/nCYGSLbZGAo8OnHnye2F5DX0vvlZKnHSzloJsnVLy1nK8J\n2J4/GoaJfAH5wqD43Gwk3myke03M+vyt66hJuka+Jl/rVKaVDlm1g+5eq3VNQ8DfF+QLg+Mj3yi6\nPbTON3qNCF0Q07SDLrZ5+fqRQVHO1yLf7DUBkC+chJ42kSZVPwAzumfN0nUSsXUri2ay+V/17eyv\ntT+DPzTnm20p9tEuDXOgF+QLR6enV0MpZSvZqOWjv2c9e3/8+LGdVqHN0bXKIaow8GVfdm2ThrND\no1u/ky2TLuKFHpAvHJWeKgN7rE1xsubndo4GYk6n072ZbKXs9ms4tFmO5nI1vaBjgKKaXkbBw6Eg\nXzg6tSY5emjkay0iLaXgx/9Y68iHh4cw8vWTKTSPq01yfMOcqFlOJGGVrxevbiUm8oVekC8cnVqD\nHN8sx1IMJlydQhxNJNYhmRb5evlGncq0UY6eo7xuVNFgaQdttEOuF/4E5AsnISsh03s+8o2mEats\n9fADMX2Vg/ZV8FMk9Lq2cy2KfH1j9CjtgIChB+QLRyXK82YNc6LI12Rr6QVLNWgKwv6sVjxkka9J\nUruT6aKZF28mXcv9ZikMIl84FOQLJ6GnYY5OotC0g0nXqht+/PhRptPpVtR22GKdpR10c0UW+VpZ\nmZWYtSodvITtOVW4+picL/SCfOHoZLneqFmORr6a27W5bHZMJpOtqH0NsI98tZRM0wO6bdiO2mKb\nXtufj5qjR83SAVogXzgJUc8G3/5Rc76WVtBpxDaR+OfPn2UymaQ73uzs63w1NWBRrx/1HqUdaumH\nWpMcxAuHgHzhqBzSLCeLfC3PqxOJdSimX7zTw/CdyjTtEInXnyMBLxaLam8G+jXAISBfODo+2vVR\nqh22eNZz3NzcVEvYfJtITQP4jmW+WbpvC5nV8kb9GhAtvBfkC0dFG6Jrjwbfr+Hy8jLt06B/ptYc\nvUWUfvCRsN8qrGVjVDDAKUG+cFQ06lXhap8Ge2zyjcrGWuJtTarIRsJn4q1tmsgqGJAy/AnIF45K\nFPlqiZhe24YJE6/vUuaHYfZGvboA5lMPUQ7YN8np2TSBeOFPQb5wVLx8dVHNFtZMtnd3dzuRb1S3\nq8Mwa+OAPCpgrcOtRb4+7ZBFvogXjgHyhaNiMvSRr26k0A0VumvNR75Wu+sjX32dDI18o9SDyjfq\nUBZFvkgXjgnyhaOike/FxcVO2kG7lWmfhlrawSJfKyNr9QguZXdOmnY3y/K+UeRLrwY4NcgXjkpW\n7aApB98gxzfJyXK+9vz6Wv6ekUW+Udqhp0UkAoZjg3zhqNQW3DTdkI0EihbcfLrBv1507SPfaMEt\naopO5AsfBfKFo6OlZrbgZgLWbcRRe0if81UBl9I3Gy4Sr8/7RpGvitc3ykHAcGyQLxyVP4l8/UaL\nKPK112gRVTto1BvlfFsLbgDHBPlCF7Vf9/WxRax+c4XmfP2IoGiHW1TjeyjRwlsWCUedyYh64ZQg\nX2iS7SSL7mu/3Z5+DbVUw5+IF+Czg3yhSTQSKDureGsSbi20+VwvwLmBfKGL2jggjVQj8daiYBVv\nrcQM4NxAvtDER7fak9c/9mmHSMBRdYNvqKPbigHOEeQLTTTtoI3Q9WzXtYg3euzbTZLzhe8C8oUu\nfIN02zqszdGtQXrURCfK+Wq06w9yvnDuIF9okk2msIg12kbcU/VgJWU+es7qewHOCeQLTaJBmL6e\n1/ft7RkNdHV1FQ7W1HvIF84V5AtdRDnfaEqFRrxZusEOHfeeDcZEvnCuIF9oki24+X69kXBrkfAh\nY4IAzg3kC018ZKoLbtE2Yo2E/dw2/fMXFxfb56+dD8FvC24d2d8DODXIF7rw0a+WmqlQtWTM1+2q\nwPV5o3ON1ky1Q+Sb9XFgdBCcGuQLTfyCW1T14Heq+a3CWYqh9pqemgz1sW+SUzsyATM6CE4N8oUu\nojrfrOIhq9mNFtFarSJrEo4EeWjaIZpWgXjhI0C+0EQjVb+dOEs7ZJFvNAzTXiO69kSC9PcOjXxr\n6Qd9XoBjgnyhCxWvb5auVQ8+53ss8UYyzPK0h0a+tRHxiBdOBfKFJodEvr43byZge17/Or3UcrTv\niXxrlRAApwD5Qhe1Bbco5RD15s0iX3v+6DWVQ6PZllxri20IGE4N8oUmWeTrezxEfXkPEbB/zRaH\niviQagd9DYBTgHyhiqYIfNTbk3bIxNt6vRq1PO97Uwz68eg1AI4N8oUmrTrfKN1w6KJbL7XFtR7B\nHiJlgFOCfKGLmoB95YMX7rEmEftUQXb4UfB26NRiHQ3fs9kC4NggXxgcL7go5/r29rYjULv25/V6\nXZ6fn8uvX7/Kv//+W379+lUeHx/L09NTeX5+LvP5vLy8vJTlcpnKGQnDR4B8YVB8XjU7r1arrSw1\nsl0ul3v3np+fy+/fv3eOp6enMpvNynw+L4vFYk++Kl6NipEunArkC4ORLW5F1xbdmjj17O/NZrPy\n+Pi4jXgfHx/L8/NzeX5+LrPZrLy8vJTFYrGVb5aOIOqFU4J8YVBqlQt6rNfrbaRrqYPsmM1m5fn5\neZtqUPFGaYdaHhj5wqlAvvApaFUuWGphsViUl5eXMp/Py2w22wpVz/6+v6fyfX193RGu5nw17YCA\n4dggXxicTLp6rZGvRrdRhDufz7eSjc6WqrDIN6uiIPKFU4J8YVBqmyX0vFqttnldla8tplmO9/Hx\ncUewtcMW6/xrRaVnAMcG+cLgZJsh9FoX3CyCfXp6Kk9PT+XXr187lQ2z2WwbJVt6wa718evra1mv\n182NGgCnAPnCYHi51XakWc7Xpx0eHx/L79+/y7///rut653P5ztlZLXrbFtx9hjgWCBfGJyeLcCW\n89XI18v3n3/+Kf/880+Zz+d75WPZEb0BRNcAxwb5QjdRROgXyDJxRkcpZW9Tg39s11rTa1UMtthm\n6QYT8MvLS7h1ONtODDAEyBeaWLWBVhwsFotydXW17eFg/RosN6t/ziJVk+Z0Oi3T6bTZo0EF/PT0\nVP7555/y77//lt+/f283TuiONa1caG0RRrowNMgXqmhUa7nS5XK5baCjkyk2m83O4paKdzKZlNls\nViaTyfYoJZ+35sVs/RpscU3la3W7tmEiq1qwzxHgM4B8oYkJTCNfH/H6WtzFYlFubm7KZDIpNzc3\ne9c3NzellLIn2ux6Pp9vpWvlZbZrrVY2FkW+CBg+A8gXmmip1+vr606LyFJ25bxcLsv19XW5ubnZ\nNle3a3/PnrtWY2tnKy2zPK/vUqZbhbONGlQtwGcC+UITlasJ2MTr7+soIbvOzvrctVrbt7e3slgs\n9rYK+y5lfrda6wAYEuQLTTTnqzlevwFiuVzuTbKIplvY2Z6759CGOtpYJ8v5Rs8d/bsAhgL5QhPL\n547H47JarXbuaZ7Xqh90moWe/bU9T5aX1UNzyb6NZJTz9c/jrwGGBvlCE418S/kv4rXqh2hKsV3X\nHttz29kviOk93zDdN1S3ax/51s4AQ4J8ocnb29tOVYOlGnSWm85208PPfdP7RlR/6++1xgjZtb0x\n1J7bXwMMweGTDI8LPwFfAJ027KcPR/f938mOUmIJRvdqZWj+ce15avcBjkjTrcgXAOD4NN06/ojP\nAgAAdkG+AAADgHwBAAYA+QIADADyBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAM+T/ACE/7be2\nhxMvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4c8012cf90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classified as 7 with probability 0.999391\n"
     ]
    }
   ],
   "source": [
    "# @@@ AUTOTEST_OUTPUT_IGNORED_CELL\n",
    "plt.imshow(val_img[0], cmap='Greys_r')\n",
    "plt.axis('off')\n",
    "plt.show()\n",
    "prob = model.predict(val_img[0:1].astype(np.float32)/255)[0]\n",
    "assert max(prob) > 0.99, \"Low prediction accuracy.\"\n",
    "print 'Classified as %d with probability %f' % (prob.argmax(), max(prob))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also evaluate the accuracy given a data iterator. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 96.820000%\n"
     ]
    }
   ],
   "source": [
    "# @@@ AUTOTEST_OUTPUT_IGNORED_CELL\n",
    "valid_acc = model.score(val_iter)\n",
    "print 'Validation accuracy: %f%%' % (valid_acc *100,)\n",
    "assert valid_acc > 0.95, \"Low validation accuracy.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even more, we can recognizes the digit written on the below box. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style type=\"text/css\">\n",
       "  canvas { border: 1px solid black; }\n",
       "</style>\n",
       "\n",
       "<div id=\"board\">\n",
       "\n",
       "  <canvas id=\"myCanvas\" width=\"100px\" height=\"100px\">\n",
       "    Sorry, your browser doesn't support canvas technology.\n",
       "  </canvas>\n",
       "\n",
       "  <p>\n",
       "\n",
       "    <button id=\"classify\" onclick=\"classify()\">\n",
       "      Classify\n",
       "    </button>\n",
       "\n",
       "    <button id=\"clear\" onclick=\"myClear()\">\n",
       "      Clear\n",
       "    </button>\n",
       "    Result: \n",
       "    <input type=\"text\" id=\"result_output\" size=\"5\" value=\"\">\n",
       "\n",
       "  </p>\n",
       "\n",
       "</div>\n",
       "\n",
       "<script type = \"text/JavaScript\" src = \"https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js?ver=1.4.2\" > </script>\n",
       "\n",
       "<script type = \"text/javascript\" >\n",
       "\n",
       "    function init() {\n",
       "        var myCanvas = document.getElementById(\"myCanvas\");\n",
       "        var curColor = $('#selectColor option:selected').val();\n",
       "        if (myCanvas) {\n",
       "            var isDown = false;\n",
       "            var ctx = myCanvas.getContext(\"2d\");\n",
       "            var canvasX, canvasY;\n",
       "            ctx.lineWidth = 8;\n",
       "            $(myCanvas).mousedown(function(e) {\n",
       "                isDown = true;\n",
       "                ctx.beginPath();\n",
       "                var parentOffset = $(this).parent().offset();\n",
       "                canvasX = e.pageX - parentOffset.left;\n",
       "                canvasY = e.pageY - parentOffset.top;\n",
       "                ctx.moveTo(canvasX, canvasY);\n",
       "            }).mousemove(function(e) {\n",
       "                if (isDown != false) {\n",
       "                    var parentOffset = $(this).parent().offset();\n",
       "                    canvasX = e.pageX - parentOffset.left;\n",
       "                    canvasY = e.pageY - parentOffset.top;\n",
       "                    ctx.lineTo(canvasX, canvasY);\n",
       "                    ctx.strokeStyle = curColor;\n",
       "                    ctx.stroke();\n",
       "                }\n",
       "            }).mouseup(function(e) {\n",
       "                isDown = false;\n",
       "                ctx.closePath();\n",
       "            });\n",
       "        }\n",
       "        $('#selectColor').change(function() {\n",
       "            curColor = $('#selectColor option:selected').val();\n",
       "        });\n",
       "    }\n",
       "init();\n",
       "\n",
       "function handle_output(out) {\n",
       "    document.getElementById(\"result_output\").value = out.content.data[\"text/plain\"];\n",
       "}\n",
       "\n",
       "function classify() {\n",
       "    var kernel = IPython.notebook.kernel;\n",
       "    var myCanvas = document.getElementById(\"myCanvas\");\n",
       "    data = myCanvas.toDataURL('image/png');\n",
       "    document.getElementById(\"result_output\").value = \"\";\n",
       "    kernel.execute(\"classify('\" + data + \"')\", {\n",
       "        'iopub': {\n",
       "            'output': handle_output\n",
       "        }\n",
       "    }, {\n",
       "        silent: false\n",
       "    });\n",
       "}\n",
       "\n",
       "function myClear() {\n",
       "    var myCanvas = document.getElementById(\"myCanvas\");\n",
       "    myCanvas.getContext(\"2d\").clearRect(0, 0, myCanvas.width, myCanvas.height);\n",
       "}\n",
       "\n",
       "\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import HTML\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "def classify(img):\n",
    "    img = img[len('data:image/png;base64,'):].decode('base64')\n",
    "    img = cv2.imdecode(np.fromstring(img, np.uint8), -1)\n",
    "    img = cv2.resize(img[:,:,3], (28,28))\n",
    "    img = img.astype(np.float32).reshape((1,1,28,28))/255.0\n",
    "    return model.predict(img)[0].argmax()\n",
    "\n",
    "'''\n",
    "To see the model in action, run the demo notebook at\n",
    "https://github.com/dmlc/mxnet-notebooks/blob/master/python/tutorials/mnist.ipynb.\n",
    "'''\n",
    "HTML(filename=\"mnist_demo.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolutional Neural Networks\n",
    "\n",
    "Note that the previous fully-connected layer simply reshapes the image into a vector during training. It ignores the spatial information that pixels are correlated on both horizontal and vertical dimensions. The convolutional layer aims to improve this drawback by using a more structural weight $W$. Instead of simply matrix-matrix multiplication, it uses 2-D convolution to obtain the output. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://thatindiandude.github.io/images/conv.png\" style=\"height: 75%; width: 75%;\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also have multiple feature maps, each with their own weight matrices, to capture different features: \n",
    "<img src=\"https://thatindiandude.github.io/images/filters.png\" style=\"height: 75%; width: 75%;\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Besides the convolutional layer, another major change of the convolutional neural network is the adding of pooling layers. A pooling layer reduce a $n\\times m$ (often called kernal size) image patch into a single value to make the network less sensitive to the spatial location.\n",
    "\n",
    "<img src=\"https://thatindiandude.github.io/images/pooling.png\" style=\"height: 75%; width: 75%;\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.36.0 (20140111.2315)\n",
       " -->\n",
       "<!-- Title: plot Pages: 1 -->\n",
       "<svg width=\"214pt\" height=\"1276pt\"\n",
       " viewBox=\"0.00 0.00 214.00 1276.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1272)\">\n",
       "<title>plot</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-1272 210,-1272 210,4 -4,4\"/>\n",
       "<!-- data -->\n",
       "<g id=\"node1\" class=\"node\"><title>data</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"47\" cy=\"-29\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-25.3\" font-family=\"Times,serif\" font-size=\"14.00\">data</text>\n",
       "</g>\n",
       "<!-- convolution0 -->\n",
       "<g id=\"node2\" class=\"node\"><title>convolution0</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-168 -7.10543e-15,-168 -7.10543e-15,-110 94,-110 94,-168\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-142.8\" font-family=\"Times,serif\" font-size=\"14.00\">Convolution</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-127.8\" font-family=\"Times,serif\" font-size=\"14.00\">5x5/1, 20</text>\n",
       "</g>\n",
       "<!-- convolution0&#45;&gt;data -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>convolution0&#45;&gt;data</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-99.8131C47,-86.1516 47,-71.0092 47,-58.3283\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-109.906 42.5001,-99.9062 47,-104.906 47.0001,-99.9062 47.0001,-99.9062 47.0001,-99.9062 47,-104.906 51.5001,-99.9062 47,-109.906 47,-109.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"71\" y=\"-80.3\" font-family=\"Times,serif\" font-size=\"14.00\">1x28x28</text>\n",
       "</g>\n",
       "<!-- activation0 -->\n",
       "<g id=\"node3\" class=\"node\"><title>activation0</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-278 -7.10543e-15,-278 -7.10543e-15,-220 94,-220 94,-278\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-252.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-237.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- activation0&#45;&gt;convolution0 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>activation0&#45;&gt;convolution0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-209.813C47,-196.152 47,-181.009 47,-168.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-219.906 42.5001,-209.906 47,-214.906 47.0001,-209.906 47.0001,-209.906 47.0001,-209.906 47,-214.906 51.5001,-209.906 47,-219.906 47,-219.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"74.5\" y=\"-190.3\" font-family=\"Times,serif\" font-size=\"14.00\">20x24x24</text>\n",
       "</g>\n",
       "<!-- pooling0 -->\n",
       "<g id=\"node4\" class=\"node\"><title>pooling0</title>\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-388 -7.10543e-15,-388 -7.10543e-15,-330 94,-330 94,-388\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-362.8\" font-family=\"Times,serif\" font-size=\"14.00\">Pooling</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-347.8\" font-family=\"Times,serif\" font-size=\"14.00\">max, 2x2/2</text>\n",
       "</g>\n",
       "<!-- pooling0&#45;&gt;activation0 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>pooling0&#45;&gt;activation0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-319.813C47,-306.152 47,-291.009 47,-278.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-329.906 42.5001,-319.906 47,-324.906 47.0001,-319.906 47.0001,-319.906 47.0001,-319.906 47,-324.906 51.5001,-319.906 47,-329.906 47,-329.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"74.5\" y=\"-300.3\" font-family=\"Times,serif\" font-size=\"14.00\">20x24x24</text>\n",
       "</g>\n",
       "<!-- convolution1 -->\n",
       "<g id=\"node5\" class=\"node\"><title>convolution1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-498 -7.10543e-15,-498 -7.10543e-15,-440 94,-440 94,-498\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-472.8\" font-family=\"Times,serif\" font-size=\"14.00\">Convolution</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-457.8\" font-family=\"Times,serif\" font-size=\"14.00\">5x5/1, 50</text>\n",
       "</g>\n",
       "<!-- convolution1&#45;&gt;pooling0 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>convolution1&#45;&gt;pooling0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-429.813C47,-416.152 47,-401.009 47,-388.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-439.906 42.5001,-429.906 47,-434.906 47.0001,-429.906 47.0001,-429.906 47.0001,-429.906 47,-434.906 51.5001,-429.906 47,-439.906 47,-439.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"74.5\" y=\"-410.3\" font-family=\"Times,serif\" font-size=\"14.00\">20x12x12</text>\n",
       "</g>\n",
       "<!-- activation1 -->\n",
       "<g id=\"node6\" class=\"node\"><title>activation1</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-608 -7.10543e-15,-608 -7.10543e-15,-550 94,-550 94,-608\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-582.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-567.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- activation1&#45;&gt;convolution1 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>activation1&#45;&gt;convolution1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-539.813C47,-526.152 47,-511.009 47,-498.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-549.906 42.5001,-539.906 47,-544.906 47.0001,-539.906 47.0001,-539.906 47.0001,-539.906 47,-544.906 51.5001,-539.906 47,-549.906 47,-549.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-520.3\" font-family=\"Times,serif\" font-size=\"14.00\">50x8x8</text>\n",
       "</g>\n",
       "<!-- pooling1 -->\n",
       "<g id=\"node7\" class=\"node\"><title>pooling1</title>\n",
       "<polygon fill=\"#80b1d3\" stroke=\"black\" points=\"94,-718 -7.10543e-15,-718 -7.10543e-15,-660 94,-660 94,-718\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-692.8\" font-family=\"Times,serif\" font-size=\"14.00\">Pooling</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-677.8\" font-family=\"Times,serif\" font-size=\"14.00\">max, 2x2/2</text>\n",
       "</g>\n",
       "<!-- pooling1&#45;&gt;activation1 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>pooling1&#45;&gt;activation1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-649.813C47,-636.152 47,-621.009 47,-608.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-659.906 42.5001,-649.906 47,-654.906 47.0001,-649.906 47.0001,-649.906 47.0001,-649.906 47,-654.906 51.5001,-649.906 47,-659.906 47,-659.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-630.3\" font-family=\"Times,serif\" font-size=\"14.00\">50x8x8</text>\n",
       "</g>\n",
       "<!-- flatten1 -->\n",
       "<g id=\"node8\" class=\"node\"><title>flatten1</title>\n",
       "<polygon fill=\"#fdb462\" stroke=\"black\" points=\"94,-828 -7.10543e-15,-828 -7.10543e-15,-770 94,-770 94,-828\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-795.3\" font-family=\"Times,serif\" font-size=\"14.00\">Flatten</text>\n",
       "</g>\n",
       "<!-- flatten1&#45;&gt;pooling1 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>flatten1&#45;&gt;pooling1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-759.813C47,-746.152 47,-731.009 47,-718.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-769.906 42.5001,-759.906 47,-764.906 47.0001,-759.906 47.0001,-759.906 47.0001,-759.906 47,-764.906 51.5001,-759.906 47,-769.906 47,-769.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-740.3\" font-family=\"Times,serif\" font-size=\"14.00\">50x4x4</text>\n",
       "</g>\n",
       "<!-- fullyconnected0 -->\n",
       "<g id=\"node9\" class=\"node\"><title>fullyconnected0</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-938 -7.10543e-15,-938 -7.10543e-15,-880 94,-880 94,-938\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-912.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-897.8\" font-family=\"Times,serif\" font-size=\"14.00\">500</text>\n",
       "</g>\n",
       "<!-- fullyconnected0&#45;&gt;flatten1 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>fullyconnected0&#45;&gt;flatten1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-869.813C47,-856.152 47,-841.009 47,-828.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-879.906 42.5001,-869.906 47,-874.906 47.0001,-869.906 47.0001,-869.906 47.0001,-869.906 47,-874.906 51.5001,-869.906 47,-879.906 47,-879.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-850.3\" font-family=\"Times,serif\" font-size=\"14.00\">800</text>\n",
       "</g>\n",
       "<!-- activation2 -->\n",
       "<g id=\"node10\" class=\"node\"><title>activation2</title>\n",
       "<polygon fill=\"#ffffb3\" stroke=\"black\" points=\"94,-1048 -7.10543e-15,-1048 -7.10543e-15,-990 94,-990 94,-1048\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1022.8\" font-family=\"Times,serif\" font-size=\"14.00\">Activation</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1007.8\" font-family=\"Times,serif\" font-size=\"14.00\">tanh</text>\n",
       "</g>\n",
       "<!-- activation2&#45;&gt;fullyconnected0 -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>activation2&#45;&gt;fullyconnected0</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-979.813C47,-966.152 47,-951.009 47,-938.328\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-989.906 42.5001,-979.906 47,-984.906 47.0001,-979.906 47.0001,-979.906 47.0001,-979.906 47,-984.906 51.5001,-979.906 47,-989.906 47,-989.906\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-960.3\" font-family=\"Times,serif\" font-size=\"14.00\">500</text>\n",
       "</g>\n",
       "<!-- fullyconnected1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>fullyconnected1</title>\n",
       "<polygon fill=\"#fb8072\" stroke=\"black\" points=\"94,-1158 -7.10543e-15,-1158 -7.10543e-15,-1100 94,-1100 94,-1158\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1132.8\" font-family=\"Times,serif\" font-size=\"14.00\">FullyConnected</text>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1117.8\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- fullyconnected1&#45;&gt;activation2 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>fullyconnected1&#45;&gt;activation2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47,-1089.81C47,-1076.15 47,-1061.01 47,-1048.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"47,-1099.91 42.5001,-1089.91 47,-1094.91 47.0001,-1089.91 47.0001,-1089.91 47.0001,-1089.91 47,-1094.91 51.5001,-1089.91 47,-1099.91 47,-1099.91\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.5\" y=\"-1070.3\" font-family=\"Times,serif\" font-size=\"14.00\">500</text>\n",
       "</g>\n",
       "<!-- softmax_label -->\n",
       "<g id=\"node12\" class=\"node\"><title>softmax_label</title>\n",
       "<ellipse fill=\"#8dd3c7\" stroke=\"black\" cx=\"159\" cy=\"-1129\" rx=\"47\" ry=\"29\"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-1125.3\" font-family=\"Times,serif\" font-size=\"14.00\">softmax_label</text>\n",
       "</g>\n",
       "<!-- softmax -->\n",
       "<g id=\"node13\" class=\"node\"><title>softmax</title>\n",
       "<polygon fill=\"#fccde5\" stroke=\"black\" points=\"170,-1268 76,-1268 76,-1210 170,-1210 170,-1268\"/>\n",
       "<text text-anchor=\"middle\" x=\"123\" y=\"-1235.3\" font-family=\"Times,serif\" font-size=\"14.00\">SoftmaxOutput</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;fullyconnected1 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>softmax&#45;&gt;fullyconnected1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M97.2538,-1201.41C87.3525,-1187.34 76.2083,-1171.51 66.9347,-1158.33\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.23,-1209.91 93.7952,-1204.32 100.353,-1205.82 97.4753,-1201.73 97.4753,-1201.73 97.4753,-1201.73 100.353,-1205.82 101.155,-1199.14 103.23,-1209.91 103.23,-1209.91\"/>\n",
       "<text text-anchor=\"middle\" x=\"97\" y=\"-1180.3\" font-family=\"Times,serif\" font-size=\"14.00\">10</text>\n",
       "</g>\n",
       "<!-- softmax&#45;&gt;softmax_label -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>softmax&#45;&gt;softmax_label</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M135.648,-1200.06C140.313,-1186.06 145.505,-1170.48 149.807,-1157.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"132.365,-1209.91 131.258,-1199 133.946,-1205.16 135.527,-1200.42 135.527,-1200.42 135.527,-1200.42 133.946,-1205.16 139.796,-1201.84 132.365,-1209.91 132.365,-1209.91\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f4c2be99390>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = mx.symbol.Variable('data')\n",
    "# first conv layer\n",
    "conv1 = mx.sym.Convolution(data=data, kernel=(5,5), num_filter=20)\n",
    "tanh1 = mx.sym.Activation(data=conv1, act_type=\"tanh\")\n",
    "pool1 = mx.sym.Pooling(data=tanh1, pool_type=\"max\", kernel=(2,2), stride=(2,2))\n",
    "# second conv layer\n",
    "conv2 = mx.sym.Convolution(data=pool1, kernel=(5,5), num_filter=50)\n",
    "tanh2 = mx.sym.Activation(data=conv2, act_type=\"tanh\")\n",
    "pool2 = mx.sym.Pooling(data=tanh2, pool_type=\"max\", kernel=(2,2), stride=(2,2))\n",
    "# first fullc layer\n",
    "flatten = mx.sym.Flatten(data=pool2)\n",
    "fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=500)\n",
    "tanh3 = mx.sym.Activation(data=fc1, act_type=\"tanh\")\n",
    "# second fullc\n",
    "fc2 = mx.sym.FullyConnected(data=tanh3, num_hidden=10)\n",
    "# softmax loss\n",
    "lenet = mx.sym.SoftmaxOutput(data=fc2, name='softmax')\n",
    "mx.viz.plot_network(symbol=lenet, shape=shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that LeNet is more complex than the previous multilayer perceptron, so we use GPU instead of CPU for training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:\u001b[91m[Deprecation Warning] mxnet.model.FeedForward has been deprecated. Please use mxnet.mod.Module instead.\u001b[0m\n",
      "INFO:root:Start training with [gpu(0)]\n",
      "INFO:root:Epoch[0] Batch [200]\tSpeed: 21246.94 samples/sec\tTrain-accuracy=0.111550\n",
      "INFO:root:Epoch[0] Batch [400]\tSpeed: 23678.45 samples/sec\tTrain-accuracy=0.113800\n",
      "INFO:root:Epoch[0] Batch [600]\tSpeed: 23676.68 samples/sec\tTrain-accuracy=0.110600\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=2.639\n",
      "INFO:root:Epoch[0] Validation-accuracy=0.113500\n",
      "INFO:root:Epoch[1] Batch [200]\tSpeed: 23831.23 samples/sec\tTrain-accuracy=0.149400\n",
      "INFO:root:Epoch[1] Batch [400]\tSpeed: 23667.46 samples/sec\tTrain-accuracy=0.780650\n",
      "INFO:root:Epoch[1] Batch [600]\tSpeed: 23671.74 samples/sec\tTrain-accuracy=0.912300\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=2.535\n",
      "INFO:root:Epoch[1] Validation-accuracy=0.935700\n",
      "INFO:root:Epoch[2] Batch [200]\tSpeed: 23709.24 samples/sec\tTrain-accuracy=0.942450\n",
      "INFO:root:Epoch[2] Batch [400]\tSpeed: 23602.33 samples/sec\tTrain-accuracy=0.957550\n",
      "INFO:root:Epoch[2] Batch [600]\tSpeed: 23586.94 samples/sec\tTrain-accuracy=0.964650\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=2.545\n",
      "INFO:root:Epoch[2] Validation-accuracy=0.970200\n",
      "INFO:root:Epoch[3] Batch [200]\tSpeed: 23784.07 samples/sec\tTrain-accuracy=0.971200\n",
      "INFO:root:Epoch[3] Batch [400]\tSpeed: 23692.30 samples/sec\tTrain-accuracy=0.975100\n",
      "INFO:root:Epoch[3] Batch [600]\tSpeed: 23679.82 samples/sec\tTrain-accuracy=0.976250\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=2.535\n",
      "INFO:root:Epoch[3] Validation-accuracy=0.979500\n",
      "INFO:root:Epoch[4] Batch [200]\tSpeed: 23785.40 samples/sec\tTrain-accuracy=0.979900\n",
      "INFO:root:Epoch[4] Batch [400]\tSpeed: 23686.64 samples/sec\tTrain-accuracy=0.981900\n",
      "INFO:root:Epoch[4] Batch [600]\tSpeed: 23704.04 samples/sec\tTrain-accuracy=0.980700\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=2.535\n",
      "INFO:root:Epoch[4] Validation-accuracy=0.982500\n",
      "INFO:root:Epoch[5] Batch [200]\tSpeed: 23740.93 samples/sec\tTrain-accuracy=0.984100\n",
      "INFO:root:Epoch[5] Batch [400]\tSpeed: 23670.94 samples/sec\tTrain-accuracy=0.986350\n",
      "INFO:root:Epoch[5] Batch [600]\tSpeed: 23675.28 samples/sec\tTrain-accuracy=0.984400\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=2.538\n",
      "INFO:root:Epoch[5] Validation-accuracy=0.984600\n",
      "INFO:root:Epoch[6] Batch [200]\tSpeed: 23825.26 samples/sec\tTrain-accuracy=0.986750\n",
      "INFO:root:Epoch[6] Batch [400]\tSpeed: 23705.14 samples/sec\tTrain-accuracy=0.988550\n",
      "INFO:root:Epoch[6] Batch [600]\tSpeed: 23710.37 samples/sec\tTrain-accuracy=0.986350\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=2.532\n",
      "INFO:root:Epoch[6] Validation-accuracy=0.986400\n",
      "INFO:root:Epoch[7] Batch [200]\tSpeed: 23811.82 samples/sec\tTrain-accuracy=0.988800\n",
      "INFO:root:Epoch[7] Batch [400]\tSpeed: 23713.21 samples/sec\tTrain-accuracy=0.990200\n",
      "INFO:root:Epoch[7] Batch [600]\tSpeed: 23710.62 samples/sec\tTrain-accuracy=0.988350\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=2.532\n",
      "INFO:root:Epoch[7] Validation-accuracy=0.987700\n",
      "INFO:root:Epoch[8] Batch [200]\tSpeed: 23741.21 samples/sec\tTrain-accuracy=0.990350\n",
      "INFO:root:Epoch[8] Batch [400]\tSpeed: 23655.82 samples/sec\tTrain-accuracy=0.991600\n",
      "INFO:root:Epoch[8] Batch [600]\tSpeed: 23653.22 samples/sec\tTrain-accuracy=0.989900\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=2.539\n",
      "INFO:root:Epoch[8] Validation-accuracy=0.987600\n",
      "INFO:root:Epoch[9] Batch [200]\tSpeed: 23753.31 samples/sec\tTrain-accuracy=0.991800\n",
      "INFO:root:Epoch[9] Batch [400]\tSpeed: 23663.07 samples/sec\tTrain-accuracy=0.992850\n",
      "INFO:root:Epoch[9] Batch [600]\tSpeed: 23688.74 samples/sec\tTrain-accuracy=0.990850\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=2.537\n",
      "INFO:root:Epoch[9] Validation-accuracy=0.988300\n"
     ]
    }
   ],
   "source": [
    "# @@@ AUTOTEST_OUTPUT_IGNORED_CELL\n",
    "model = mx.model.FeedForward(\n",
    "    ctx = mx.gpu(0),     # use GPU 0 for training, others are same as before\n",
    "    symbol = lenet,       \n",
    "    num_epoch = 10,     \n",
    "    learning_rate = 0.1)\n",
    "model.fit(\n",
    "    X=train_iter,  \n",
    "    eval_data=val_iter, \n",
    "    batch_end_callback = mx.callback.Speedometer(batch_size, 200)\n",
    ") \n",
    "assert model.score(val_iter) > 0.98, \"Low validation accuracy.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that, with the same hyper-parameters, LeNet achieves 98.7% validation accuracy, which improves on the previous multilayer perceptron accuracy of 96.6%.\n",
    "\n",
    "Because we rewrite the model parameters in `mod`, now we can try the previous digit recognition box again to check if or not the new CNN model improves the classification accuracy."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
